Repository: matrix-org/synapse

Issue:
User directory background process gets stuck when someone sets a name with a null codepoint
Reported by @Fizzadar via #12743 and @andrewsh via chat.

An event that looks like this:

```json
{
    "content":
    {
        "displayname": "\u0001VERSION\u0001\u0000",
        "membership": "join"
    }
}
```

Causes an exception updating the user directory table `A string literal cannot contain NUL (0x00) characters` which never passes and just keeps retrying consuming a lot of CPU on main.

```
2022-05-17 11:18:34,417 - synapse.metrics.background_process_metrics - 245 - ERROR - user_directory.notify_new_event-2101 - Background process 'user_directory.notify_new_event' threw an exception
Traceback (most recent call last):                                                                                                                                                                                                             
  File "/opt/venvs/matrix-synapse/lib/python3.9/site-packages/synapse/metrics/background_process_metrics.py", line 243, in run
    return await func(*args, **kwargs)                                                                                                                                                                                                         
  File "/opt/venvs/matrix-synapse/lib/python3.9/site-packages/synapse/handlers/user_directory.py", line 119, in process 
    await self._unsafe_process()                                                                                                                                                                                                               
  File "/opt/venvs/matrix-synapse/lib/python3.9/site-packages/synapse/handlers/user_directory.py", line 182, in _unsafe_process
    await self._handle_deltas(deltas)                                                                                                                                                                                                          
  File "/opt/venvs/matrix-synapse/lib/python3.9/site-packages/synapse/handlers/user_directory.py", line 211, in _handle_deltas
    await self._handle_room_membership_event(                                                                                                                                                                                                  
  File "/opt/venvs/matrix-synapse/lib/python3.9/site-packages/synapse/handlers/user_directory.py", line 348, in _handle_room_membership_event
    await self._handle_possible_remote_profile_change(                                                                                                                                                                                         
  File "/opt/venvs/matrix-synapse/lib/python3.9/site-packages/synapse/handlers/user_directory.py", line 478, in _handle_possible_remote_profile_change
    await self.store.update_profile_in_user_dir(user_id, new_name, new_avatar)                                                                                                                                                                 
  File "/opt/venvs/matrix-synapse/lib/python3.9/site-packages/synapse/storage/databases/main/user_directory.py", line 522, in update_profile_in_user_dir
    await self.db_pool.runInteraction(                                                                                                                                                                                                         
  File "/opt/venvs/matrix-synapse/lib/python3.9/site-packages/synapse/storage/database.py", line 814, in runInteraction 
    return await delay_cancellation(_runInteraction())                                                                                                                                                                                         
  File "/opt/venvs/matrix-synapse/lib/python3.9/site-packages/twisted/internet/defer.py", line 1656, in _inlineCallbacks
    result = current_context.run(                                                                                                                                                                                                              
  File "/opt/venvs/matrix-synapse/lib/python3.9/site-packages/twisted/python/failure.py", line 514, in throwExceptionIntoGenerator
    return g.throw(self.type, self.value, self.tb)                                                                                                                                                                                             
  File "/opt/venvs/matrix-synapse/lib/python3.9/site-packages/synapse/storage/database.py", line 786, in _runInteraction
    result = await self.runWithConnection(                                                                                                                                                                                                     
  File "/opt/venvs/matrix-synapse/lib/python3.9/site-packages/synapse/storage/database.py", line 909, in runWithConnection
    return await make_deferred_yieldable(                                                                              
  File "/opt/venvs/matrix-synapse/lib/python3.9/site-packages/twisted/python/threadpool.py", line 244, in inContext
    result = inContext.theWork()  # type: ignore[attr-defined]                                                    
  File "/opt/venvs/matrix-synapse/lib/python3.9/site-packages/twisted/python/threadpool.py", line 260, in <lambda>
    inContext.theWork = lambda: context.call(  # type: ignore[attr-defined]                                           
  File "/opt/venvs/matrix-synapse/lib/python3.9/site-packages/twisted/python/context.py", line 117, in callWithContext
    return self.currentContext().callWithContext(ctx, func, *args, **kw)                                             
  File "/opt/venvs/matrix-synapse/lib/python3.9/site-packages/twisted/python/context.py", line 82, in callWithContext
    return func(*args, **kw)                                                                                                                                                                                                                   
  File "/opt/venvs/matrix-synapse/lib/python3.9/site-packages/twisted/enterprise/adbapi.py", line 282, in _runWithConnection
    result = func(conn, *args, **kw)                                                                                   
  File "/opt/venvs/matrix-synapse/lib/python3.9/site-packages/synapse/storage/database.py", line 902, in inner_func
    return func(db_conn, *args, **kwargs)                                                                                                                                                                                                      
  File "/opt/venvs/matrix-synapse/lib/python3.9/site-packages/synapse/storage/database.py", line 650, in new_transaction
    r = func(cursor, *args, **kwargs)                                                                                                                                                                                                          
  File "/opt/venvs/matrix-synapse/lib/python3.9/site-packages/synapse/storage/databases/main/user_directory.py", line 479, in _update_profile_in_user_dir_txn
    self.db_pool.simple_upsert_txn(                                                                                                                                                                                                            
  File "/opt/venvs/matrix-synapse/lib/python3.9/site-packages/synapse/storage/database.py", line 1150, in simple_upsert_txn
    return self.simple_upsert_txn_native_upsert(                                                                                                                                                                                               
  File "/opt/venvs/matrix-synapse/lib/python3.9/site-packages/synapse/storage/database.py", line 1276, in simple_upsert_txn_native_upsert
    txn.execute(sql, list(allvalues.values()))                                                                         
  File "/opt/venvs/matrix-synapse/lib/python3.9/site-packages/synapse/storage/database.py", line 337, in execute
    self._do_execute(self.txn.execute, sql, *args)                                                                     
  File "/opt/venvs/matrix-synapse/lib/python3.9/site-packages/synapse/storage/database.py", line 379, in _do_execute
    return func(sql, *args, **kwargs)                                                                                  
ValueError: A string literal cannot contain NUL (0x00) characters.
```



Discussion:

Patch:
diff --git a/changelog.d/12623.feature b/changelog.d/12623.feature
deleted file mode 100644
index cdee19faf..000000000
--- a/changelog.d/12623.feature
+++ /dev/null
@@ -1,1 +0,0 @@
-Add support for [MSC3787: Allowing knocks to restricted rooms](https://github.com/matrix-org/matrix-spec-proposals/pull/3787).
\ No newline at end of file
diff --git a/changelog.d/12680.misc b/changelog.d/12680.misc
deleted file mode 100644
index dfd1f0a6c..000000000
--- a/changelog.d/12680.misc
+++ /dev/null
@@ -1,1 +0,0 @@
-Remove code which updates unused database column `application_services_state.last_txn`.
diff --git a/changelog.d/12721.bugfix b/changelog.d/12721.bugfix
deleted file mode 100644
index 6987f7ab1..000000000
--- a/changelog.d/12721.bugfix
+++ /dev/null
@@ -1,1 +0,0 @@
-Fix push to dismiss notifications when read on another client. Contributed by @SpiritCroc @ Beeper.
diff --git a/changelog.d/12762.misc b/changelog.d/12762.misc
new file mode 100644
index 000000000..990fb6fe7
--- /dev/null
+++ b/changelog.d/12762.misc
@@ -0,0 +1,1 @@
+Fix a long-standing bug where the user directory background process would fail to make forward progress if a user included a null codepoint in their display name or avatar.
diff --git a/synapse/api/constants.py b/synapse/api/constants.py
index 330de21f6..0ccd4c955 100644
--- a/synapse/api/constants.py
+++ b/synapse/api/constants.py
@@ -65,8 +65,6 @@ class JoinRules:
     PRIVATE: Final = "private"
     # As defined for MSC3083.
     RESTRICTED: Final = "restricted"
-    # As defined for MSC3787.
-    KNOCK_RESTRICTED: Final = "knock_restricted"
 
 
 class RestrictedJoinRuleTypes:
diff --git a/synapse/api/room_versions.py b/synapse/api/room_versions.py
index 3f85d61b4..a747a4081 100644
--- a/synapse/api/room_versions.py
+++ b/synapse/api/room_versions.py
@@ -81,9 +81,6 @@ class RoomVersion:
     msc2716_historical: bool
     # MSC2716: Adds support for redacting "insertion", "chunk", and "marker" events
     msc2716_redactions: bool
-    # MSC3787: Adds support for a `knock_restricted` join rule, mixing concepts of
-    # knocks and restricted join rules into the same join condition.
-    msc3787_knock_restricted_join_rule: bool
 
 
 class RoomVersions:
@@ -102,7 +99,6 @@ class RoomVersions:
         msc2403_knocking=False,
         msc2716_historical=False,
         msc2716_redactions=False,
-        msc3787_knock_restricted_join_rule=False,
     )
     V2 = RoomVersion(
         "2",
@@ -119,7 +115,6 @@ class RoomVersions:
         msc2403_knocking=False,
         msc2716_historical=False,
         msc2716_redactions=False,
-        msc3787_knock_restricted_join_rule=False,
     )
     V3 = RoomVersion(
         "3",
@@ -136,7 +131,6 @@ class RoomVersions:
         msc2403_knocking=False,
         msc2716_historical=False,
         msc2716_redactions=False,
-        msc3787_knock_restricted_join_rule=False,
     )
     V4 = RoomVersion(
         "4",
@@ -153,7 +147,6 @@ class RoomVersions:
         msc2403_knocking=False,
         msc2716_historical=False,
         msc2716_redactions=False,
-        msc3787_knock_restricted_join_rule=False,
     )
     V5 = RoomVersion(
         "5",
@@ -170,7 +163,6 @@ class RoomVersions:
         msc2403_knocking=False,
         msc2716_historical=False,
         msc2716_redactions=False,
-        msc3787_knock_restricted_join_rule=False,
     )
     V6 = RoomVersion(
         "6",
@@ -187,7 +179,6 @@ class RoomVersions:
         msc2403_knocking=False,
         msc2716_historical=False,
         msc2716_redactions=False,
-        msc3787_knock_restricted_join_rule=False,
     )
     MSC2176 = RoomVersion(
         "org.matrix.msc2176",
@@ -204,7 +195,6 @@ class RoomVersions:
         msc2403_knocking=False,
         msc2716_historical=False,
         msc2716_redactions=False,
-        msc3787_knock_restricted_join_rule=False,
     )
     V7 = RoomVersion(
         "7",
@@ -221,7 +211,6 @@ class RoomVersions:
         msc2403_knocking=True,
         msc2716_historical=False,
         msc2716_redactions=False,
-        msc3787_knock_restricted_join_rule=False,
     )
     V8 = RoomVersion(
         "8",
@@ -238,7 +227,6 @@ class RoomVersions:
         msc2403_knocking=True,
         msc2716_historical=False,
         msc2716_redactions=False,
-        msc3787_knock_restricted_join_rule=False,
     )
     V9 = RoomVersion(
         "9",
@@ -255,7 +243,6 @@ class RoomVersions:
         msc2403_knocking=True,
         msc2716_historical=False,
         msc2716_redactions=False,
-        msc3787_knock_restricted_join_rule=False,
     )
     MSC2716v3 = RoomVersion(
         "org.matrix.msc2716v3",
@@ -272,24 +259,6 @@ class RoomVersions:
         msc2403_knocking=True,
         msc2716_historical=True,
         msc2716_redactions=True,
-        msc3787_knock_restricted_join_rule=False,
-    )
-    MSC3787 = RoomVersion(
-        "org.matrix.msc3787",
-        RoomDisposition.UNSTABLE,
-        EventFormatVersions.V3,
-        StateResolutionVersions.V2,
-        enforce_key_validity=True,
-        special_case_aliases_auth=False,
-        strict_canonicaljson=True,
-        limit_notifications_power_levels=True,
-        msc2176_redaction_rules=False,
-        msc3083_join_rules=True,
-        msc3375_redaction_rules=True,
-        msc2403_knocking=True,
-        msc2716_historical=False,
-        msc2716_redactions=False,
-        msc3787_knock_restricted_join_rule=True,
     )
 
 
@@ -307,7 +276,6 @@ KNOWN_ROOM_VERSIONS: Dict[str, RoomVersion] = {
         RoomVersions.V8,
         RoomVersions.V9,
         RoomVersions.MSC2716v3,
-        RoomVersions.MSC3787,
     )
 }
 
diff --git a/synapse/event_auth.py b/synapse/event_auth.py
index 4c0b587a7..621a3efcc 100644
--- a/synapse/event_auth.py
+++ b/synapse/event_auth.py
@@ -414,12 +414,7 @@ def _is_membership_change_allowed(
             raise AuthError(403, "You are banned from this room")
         elif join_rule == JoinRules.PUBLIC:
             pass
-        elif (
-            room_version.msc3083_join_rules and join_rule == JoinRules.RESTRICTED
-        ) or (
-            room_version.msc3787_knock_restricted_join_rule
-            and join_rule == JoinRules.KNOCK_RESTRICTED
-        ):
+        elif room_version.msc3083_join_rules and join_rule == JoinRules.RESTRICTED:
             # This is the same as public, but the event must contain a reference
             # to the server who authorised the join. If the event does not contain
             # the proper content it is rejected.
@@ -445,13 +440,8 @@ def _is_membership_change_allowed(
                 if authorising_user_level < invite_level:
                     raise AuthError(403, "Join event authorised by invalid server.")
 
-        elif (
-            join_rule == JoinRules.INVITE
-            or (room_version.msc2403_knocking and join_rule == JoinRules.KNOCK)
-            or (
-                room_version.msc3787_knock_restricted_join_rule
-                and join_rule == JoinRules.KNOCK_RESTRICTED
-            )
+        elif join_rule == JoinRules.INVITE or (
+            room_version.msc2403_knocking and join_rule == JoinRules.KNOCK
         ):
             if not caller_in_room and not caller_invited:
                 raise AuthError(403, "You are not invited to this room.")
@@ -472,10 +462,7 @@ def _is_membership_change_allowed(
         if user_level < ban_level or user_level <= target_level:
             raise AuthError(403, "You don't have permission to ban")
     elif room_version.msc2403_knocking and Membership.KNOCK == membership:
-        if join_rule != JoinRules.KNOCK and (
-            not room_version.msc3787_knock_restricted_join_rule
-            or join_rule != JoinRules.KNOCK_RESTRICTED
-        ):
+        if join_rule != JoinRules.KNOCK:
             raise AuthError(403, "You don't have permission to knock")
         elif target_user_id != event.user_id:
             raise AuthError(403, "You cannot knock for other users")
diff --git a/synapse/handlers/event_auth.py b/synapse/handlers/event_auth.py
index 6bed46435..d441ebb0a 100644
--- a/synapse/handlers/event_auth.py
+++ b/synapse/handlers/event_auth.py
@@ -241,15 +241,7 @@ class EventAuthHandler:
 
         # If the join rule is not restricted, this doesn't apply.
         join_rules_event = await self._store.get_event(join_rules_event_id)
-        content_join_rule = join_rules_event.content.get("join_rule")
-        if content_join_rule == JoinRules.RESTRICTED:
-            return True
-
-        # also check for MSC3787 behaviour
-        if room_version.msc3787_knock_restricted_join_rule:
-            return content_join_rule == JoinRules.KNOCK_RESTRICTED
-
-        return False
+        return join_rules_event.content.get("join_rule") == JoinRules.RESTRICTED
 
     async def get_rooms_that_allow_join(
         self, state_ids: StateMap[str]
diff --git a/synapse/handlers/room_summary.py b/synapse/handlers/room_summary.py
index af83de319..ff24ec806 100644
--- a/synapse/handlers/room_summary.py
+++ b/synapse/handlers/room_summary.py
@@ -562,13 +562,8 @@ class RoomSummaryHandler:
         if join_rules_event_id:
             join_rules_event = await self._store.get_event(join_rules_event_id)
             join_rule = join_rules_event.content.get("join_rule")
-            if (
-                join_rule == JoinRules.PUBLIC
-                or (room_version.msc2403_knocking and join_rule == JoinRules.KNOCK)
-                or (
-                    room_version.msc3787_knock_restricted_join_rule
-                    and join_rule == JoinRules.KNOCK_RESTRICTED
-                )
+            if join_rule == JoinRules.PUBLIC or (
+                room_version.msc2403_knocking and join_rule == JoinRules.KNOCK
             ):
                 return True
 
diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py
index d5603596c..581834452 100644
--- a/synapse/push/httppusher.py
+++ b/synapse/push/httppusher.py
@@ -405,7 +405,7 @@ class HttpPusher(Pusher):
         rejected = []
         if "rejected" in resp:
             rejected = resp["rejected"]
-        if not rejected:
+        else:
             self.badge_count_last_call = badge
         return rejected
 
diff --git a/synapse/rest/client/room.py b/synapse/rest/client/room.py
index 4b8bfbffc..5a2361a2e 100644
--- a/synapse/rest/client/room.py
+++ b/synapse/rest/client/room.py
@@ -109,10 +109,10 @@ class RoomStateEventRestServlet(TransactionRestServlet):
         self.auth = hs.get_auth()
 
     def register(self, http_server: HttpServer) -> None:
-        # /room/$roomid/state/$eventtype
+        # /rooms/$roomid/state/$eventtype
         no_state_key = "/rooms/(?P<room_id>[^/]*)/state/(?P<event_type>[^/]*)$"
 
-        # /room/$roomid/state/$eventtype/$statekey
+        # /rooms/$roomid/state/$eventtype/$statekey
         state_key = (
             "/rooms/(?P<room_id>[^/]*)/state/"
             "(?P<event_type>[^/]*)/(?P<state_key>[^/]*)$"
diff --git a/synapse/storage/databases/main/appservice.py b/synapse/storage/databases/main/appservice.py
index e284454b6..945707b0e 100644
--- a/synapse/storage/databases/main/appservice.py
+++ b/synapse/storage/databases/main/appservice.py
@@ -203,29 +203,19 @@ class ApplicationServiceTransactionWorkerStore(
         """Get the application service state.
 
         Args:
-            service: The service whose state to get.
+            service: The service whose state to set.
         Returns:
-            An ApplicationServiceState, or None if we have yet to attempt any
-            transactions to the AS.
+            An ApplicationServiceState or none.
         """
-        # if we have created transactions for this AS but not yet attempted to send
-        # them, we will have a row in the table with state=NULL (recording the stream
-        # positions we have processed up to).
-        #
-        # On the other hand, if we have yet to create any transactions for this AS at
-        # all, then there will be no row for the AS.
-        #
-        # In either case, we return None to indicate "we don't yet know the state of
-        # this AS".
-        result = await self.db_pool.simple_select_one_onecol(
+        result = await self.db_pool.simple_select_one(
             "application_services_state",
             {"as_id": service.id},
-            retcol="state",
+            ["state"],
             allow_none=True,
             desc="get_appservice_state",
         )
         if result:
-            return ApplicationServiceState(result)
+            return ApplicationServiceState(result.get("state"))
         return None
 
     async def set_appservice_state(
@@ -306,6 +296,14 @@ class ApplicationServiceTransactionWorkerStore(
         """
 
         def _complete_appservice_txn(txn: LoggingTransaction) -> None:
+            # Set current txn_id for AS to 'txn_id'
+            self.db_pool.simple_upsert_txn(
+                txn,
+                "application_services_state",
+                {"as_id": service.id},
+                {"last_txn": txn_id},
+            )
+
             # Delete txn
             self.db_pool.simple_delete_txn(
                 txn,
@@ -454,15 +452,16 @@ class ApplicationServiceTransactionWorkerStore(
                 % (stream_type,)
             )
 
-        # this may be the first time that we're recording any state for this AS, so
-        # we don't yet know if a row for it exists; hence we have to upsert here.
-        await self.db_pool.simple_upsert(
-            table="application_services_state",
-            keyvalues={"as_id": service.id},
-            values={f"{stream_type}_stream_id": pos},
-            # no need to lock when emulating upsert: as_id is a unique key
-            lock=False,
-            desc="set_appservice_stream_type_pos",
+        def set_appservice_stream_type_pos_txn(txn: LoggingTransaction) -> None:
+            stream_id_type = "%s_stream_id" % stream_type
+            txn.execute(
+                "UPDATE application_services_state SET %s = ? WHERE as_id=?"
+                % stream_id_type,
+                (pos, service.id),
+            )
+
+        await self.db_pool.runInteraction(
+            "set_appservice_stream_type_pos", set_appservice_stream_type_pos_txn
         )
 
 
diff --git a/synapse/storage/databases/main/events.py b/synapse/storage/databases/main/events.py
index 42d484dc9..0df8ff539 100644
--- a/synapse/storage/databases/main/events.py
+++ b/synapse/storage/databases/main/events.py
@@ -52,6 +52,7 @@ from synapse.storage.util.sequence import SequenceGenerator
 from synapse.types import JsonDict, StateMap, get_domain_from_id
 from synapse.util import json_encoder
 from synapse.util.iterutils import batch_iter, sorted_topologically
+from synapse.util.stringutils import non_null_str_or_none
 
 if TYPE_CHECKING:
     from synapse.server import HomeServer
@@ -1728,9 +1729,6 @@ class PersistEventsStore:
                 not affect the current local state.
         """
 
-        def non_null_str_or_none(val: Any) -> Optional[str]:
-            return val if isinstance(val, str) and "\u0000" not in val else None
-
         self.db_pool.simple_insert_many_txn(
             txn,
             table="room_memberships",
diff --git a/synapse/storage/databases/main/user_directory.py b/synapse/storage/databases/main/user_directory.py
index df772d472..028db69af 100644
--- a/synapse/storage/databases/main/user_directory.py
+++ b/synapse/storage/databases/main/user_directory.py
@@ -29,6 +29,7 @@ from typing import (
 from typing_extensions import TypedDict
 
 from synapse.api.errors import StoreError
+from synapse.util.stringutils import non_null_str_or_none
 
 if TYPE_CHECKING:
     from synapse.server import HomeServer
@@ -469,11 +470,9 @@ class UserDirectoryBackgroundUpdateStore(StateDeltasStore):
         """
         Update or add a user's profile in the user directory.
         """
-        # If the display name or avatar URL are unexpected types, overwrite them.
-        if not isinstance(display_name, str):
-            display_name = None
-        if not isinstance(avatar_url, str):
-            avatar_url = None
+        # If the display name or avatar URL are unexpected types, replace with None.
+        display_name = non_null_str_or_none(display_name)
+        avatar_url = non_null_str_or_none(avatar_url)
 
         def _update_profile_in_user_dir_txn(txn: LoggingTransaction) -> None:
             self.db_pool.simple_upsert_txn(
diff --git a/synapse/storage/schema/__init__.py b/synapse/storage/schema/__init__.py
index da98f05e0..20c344fae 100644
--- a/synapse/storage/schema/__init__.py
+++ b/synapse/storage/schema/__init__.py
@@ -61,9 +61,7 @@ Changes in SCHEMA_VERSION = 68:
 
 Changes in SCHEMA_VERSION = 69:
     - We now write to `device_lists_changes_in_room` table.
-    - We now use a PostgreSQL sequence to generate future txn_ids for
-      `application_services_txns`. `application_services_state.last_txn` is no longer
-      updated.
+    - Use sequence to generate future `application_services_txns.txn_id`s
 
 Changes in SCHEMA_VERSION = 70:
     - event_reference_hashes is no longer written to.
@@ -73,7 +71,6 @@ Changes in SCHEMA_VERSION = 70:
 SCHEMA_COMPAT_VERSION = (
     # We now assume that `device_lists_changes_in_room` has been filled out for
     # recent device_list_updates.
-    # ... and that `application_services_state.last_txn` is not used.
     69
 )
 """Limit on how far the synapse codebase can be rolled back without breaking db compat
diff --git a/synapse/util/stringutils.py b/synapse/util/stringutils.py
index b26546aec..27a363d7e 100644
--- a/synapse/util/stringutils.py
+++ b/synapse/util/stringutils.py
@@ -16,7 +16,7 @@ import itertools
 import re
 import secrets
 import string
-from typing import Iterable, Optional, Tuple
+from typing import Any, Iterable, Optional, Tuple
 
 from netaddr import valid_ipv6
 
@@ -247,3 +247,11 @@ def base62_encode(num: int, minwidth: int = 1) -> str:
     # pad to minimum width
     pad = "0" * (minwidth - len(res))
     return pad + res
+
+
+def non_null_str_or_none(val: Any) -> Optional[str]:
+    """Check that the arg is a string containing no null (U+0000) codepoints.
+
+    If so, returns the given string unmodified; otherwise, returns None.
+    """
+    return val if isinstance(val, str) and "\u0000" not in val else None


Test patch:
diff --git a/tests/handlers/test_appservice.py b/tests/handlers/test_appservice.py
index 53e7a5d81..5b0cd1ab8 100644
--- a/tests/handlers/test_appservice.py
+++ b/tests/handlers/test_appservice.py
@@ -434,6 +434,16 @@ class ApplicationServicesHandlerSendEventsTestCase(unittest.HomeserverTestCase):
             },
         )
 
+        # "Complete" a transaction.
+        # All this really does for us is make an entry in the application_services_state
+        # database table, which tracks the current stream_token per stream ID per AS.
+        self.get_success(
+            self.hs.get_datastores().main.complete_appservice_txn(
+                0,
+                interested_appservice,
+            )
+        )
+
         # Now, pretend that we receive a large burst of read receipts (300 total) that
         # all come in at once.
         for i in range(300):
diff --git a/tests/handlers/test_user_directory.py b/tests/handlers/test_user_directory.py
index 96e2e3039..4d658d29c 100644
--- a/tests/handlers/test_user_directory.py
+++ b/tests/handlers/test_user_directory.py
@@ -1007,6 +1007,34 @@ class UserDirectoryTestCase(unittest.HomeserverTestCase):
         self.assertEqual(in_public, {(bob, room1), (bob, room2)})
         self.assertEqual(in_private, set())
 
+    def test_ignore_display_names_with_null_codepoints(self) -> None:
+        MXC_DUMMY = "mxc://dummy"
+
+        # Alice creates a public room.
+        alice = self.register_user("alice", "pass")
+
+        # Alice has a user directory entry to start with.
+        self.assertIn(
+            alice,
+            self.get_success(self.user_dir_helper.get_profiles_in_user_directory()),
+        )
+
+        # Alice changes her name to include a null codepoint.
+        self.get_success(
+            self.hs.get_user_directory_handler().handle_local_profile_change(
+                alice,
+                ProfileInfo(
+                    display_name="abcd\u0000efgh",
+                    avatar_url=MXC_DUMMY,
+                ),
+            )
+        )
+        # Alice's profile should be updated with the new avatar, but no display name.
+        self.assertEqual(
+            self.get_success(self.user_dir_helper.get_profiles_in_user_directory()),
+            {alice: ProfileInfo(display_name=None, avatar_url=MXC_DUMMY)},
+        )
+
 
 class TestUserDirSearchDisabled(unittest.HomeserverTestCase):
     servlets = [
diff --git a/tests/storage/test_appservice.py b/tests/storage/test_appservice.py
index 1047ed09c..1bf93e79a 100644
--- a/tests/storage/test_appservice.py
+++ b/tests/storage/test_appservice.py
@@ -14,7 +14,7 @@
 import json
 import os
 import tempfile
-from typing import List, cast
+from typing import List, Optional, cast
 from unittest.mock import Mock
 
 import yaml
@@ -149,12 +149,15 @@ class ApplicationServiceTransactionStoreTestCase(unittest.HomeserverTestCase):
             outfile.write(yaml.dump(as_yaml))
             self.as_yaml_files.append(as_token)
 
-    def _set_state(self, id: str, state: ApplicationServiceState):
+    def _set_state(
+        self, id: str, state: ApplicationServiceState, txn: Optional[int] = None
+    ):
         return self.db_pool.runOperation(
             self.engine.convert_param_style(
-                "INSERT INTO application_services_state(as_id, state) VALUES(?,?)"
+                "INSERT INTO application_services_state(as_id, state, last_txn) "
+                "VALUES(?,?,?)"
             ),
-            (id, state.value),
+            (id, state.value, txn),
         )
 
     def _insert_txn(self, as_id, txn_id, events):
@@ -277,6 +280,17 @@ class ApplicationServiceTransactionStoreTestCase(unittest.HomeserverTestCase):
             self.store.complete_appservice_txn(txn_id=txn_id, service=service)
         )
 
+        res = self.get_success(
+            self.db_pool.runQuery(
+                self.engine.convert_param_style(
+                    "SELECT last_txn FROM application_services_state WHERE as_id=?"
+                ),
+                (service.id,),
+            )
+        )
+        self.assertEqual(1, len(res))
+        self.assertEqual(txn_id, res[0][0])
+
         res = self.get_success(
             self.db_pool.runQuery(
                 self.engine.convert_param_style(
@@ -302,13 +316,14 @@ class ApplicationServiceTransactionStoreTestCase(unittest.HomeserverTestCase):
         res = self.get_success(
             self.db_pool.runQuery(
                 self.engine.convert_param_style(
-                    "SELECT state FROM application_services_state WHERE as_id=?"
+                    "SELECT last_txn, state FROM application_services_state WHERE as_id=?"
                 ),
                 (service.id,),
             )
         )
         self.assertEqual(1, len(res))
-        self.assertEqual(ApplicationServiceState.UP.value, res[0][0])
+        self.assertEqual(txn_id, res[0][0])
+        self.assertEqual(ApplicationServiceState.UP.value, res[0][1])
 
         res = self.get_success(
             self.db_pool.runQuery(