internlm/SWE-Fixer-Train-110K huggingface dataset:
Repository: matrix-org/synapse

Issue:
Remove unstable MSC1772 identifiers
Now that we support the stable identifiers (#9915) we can at some point in the future remove support for the unstable ones. These should all appear prefixed with `org.matrix.msc1772`.


Discussion:

Resolving patch:
diff --git a/synapse/api/constants.py b/synapse/api/constants.py
--- a/synapse/api/constants.py
+++ b/synapse/api/constants.py
@@ -112,8 +112,6 @@ class EventTypes:
 
     SpaceChild = "m.space.child"
     SpaceParent = "m.space.parent"
-    MSC1772_SPACE_CHILD = "org.matrix.msc1772.space.child"
-    MSC1772_SPACE_PARENT = "org.matrix.msc1772.space.parent"
 
 
 class ToDeviceEventTypes:
@@ -180,7 +178,6 @@ class EventContentFields:
 
     # cf https://github.com/matrix-org/matrix-doc/pull/1772
     ROOM_TYPE = "type"
-    MSC1772_ROOM_TYPE = "org.matrix.msc1772.type"
 
 
 class RoomEncryptionAlgorithms:
diff --git a/synapse/handlers/space_summary.py b/synapse/handlers/space_summary.py
--- a/synapse/handlers/space_summary.py
+++ b/synapse/handlers/space_summary.py
@@ -402,10 +402,7 @@ async def _summarize_remote_room(
             return (), ()
 
         return res.rooms, tuple(
-            ev.data
-            for ev in res.events
-            if ev.event_type == EventTypes.MSC1772_SPACE_CHILD
-            or ev.event_type == EventTypes.SpaceChild
+            ev.data for ev in res.events if ev.event_type == EventTypes.SpaceChild
         )
 
     async def _is_room_accessible(
@@ -514,11 +511,6 @@ async def _build_room_entry(self, room_id: str) -> JsonDict:
             current_state_ids[(EventTypes.Create, "")]
         )
 
-        # TODO: update once MSC1772 lands
-        room_type = create_event.content.get(EventContentFields.ROOM_TYPE)
-        if not room_type:
-            room_type = create_event.content.get(EventContentFields.MSC1772_ROOM_TYPE)
-
         room_version = await self._store.get_room_version(room_id)
         allowed_spaces = None
         if await self._event_auth_handler.has_restricted_join_rules(
@@ -540,7 +532,7 @@ async def _build_room_entry(self, room_id: str) -> JsonDict:
             ),
             "guest_can_join": stats["guest_access"] == "can_join",
             "creation_ts": create_event.origin_server_ts,
-            "room_type": room_type,
+            "room_type": create_event.content.get(EventContentFields.ROOM_TYPE),
             "allowed_spaces": allowed_spaces,
         }
 
@@ -569,9 +561,7 @@ async def _get_child_events(self, room_id: str) -> Iterable[EventBase]:
             [
                 event_id
                 for key, event_id in current_state_ids.items()
-                # TODO: update once MSC1772 has been FCP for a period of time.
-                if key[0] == EventTypes.MSC1772_SPACE_CHILD
-                or key[0] == EventTypes.SpaceChild
+                if key[0] == EventTypes.SpaceChild
             ]
         )
 


Test patch:
Repository: matrix-org/synapse

Issue:
invite rejections are not sent over federation
Suppose:
 * A and B are in a room (each on a different homeserver)
 * A invites C (on a third homeserver)
 * C rejects the invite (and thus sends a leave event via `send_leave` to A)

The invite rejection *should* be sent out over federation to B, but this does not take place.


Discussion:

Resolving patch:
diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -1980,6 +1980,13 @@ async def on_send_leave_request(self, origin: str, pdu: EventBase) -> None:
 
         event.internal_metadata.outlier = False
 
+        # Send this event on behalf of the other server.
+        #
+        # The remote server isn't a full participant in the room at this point, so
+        # may not have an up-to-date list of the other homeservers participating in
+        # the room, so we send it on their behalf.
+        event.internal_metadata.send_on_behalf_of = origin
+
         context = await self.state_handler.compute_event_context(event)
         await self._auth_and_persist_event(origin, event, context)
 
@@ -2084,6 +2091,13 @@ async def on_send_knock_request(
 
         event.internal_metadata.outlier = False
 
+        # Send this event on behalf of the other server.
+        #
+        # The remote server isn't a full participant in the room at this point, so
+        # may not have an up-to-date list of the other homeservers participating in
+        # the room, so we send it on their behalf.
+        event.internal_metadata.send_on_behalf_of = origin
+
         context = await self.state_handler.compute_event_context(event)
 
         event_allowed = await self.third_party_event_rules.check_event_allowed(


Test patch:
Repository: matrix-org/synapse

Issue:
new stream_ordering column needs an `ANALYZE`
... to encourage postgres to use the index


Discussion:

Resolving patch:
diff --git a/synapse/storage/databases/main/events_bg_updates.py b/synapse/storage/databases/main/events_bg_updates.py
--- a/synapse/storage/databases/main/events_bg_updates.py
+++ b/synapse/storage/databases/main/events_bg_updates.py
@@ -1146,6 +1146,16 @@ def process(txn: Cursor) -> None:
                 logger.info("completing stream_ordering migration: %s", sql)
                 txn.execute(sql)
 
+        # ANALYZE the new column to build stats on it, to encourage PostgreSQL to use the
+        # indexes on it.
+        # We need to pass execute a dummy function to handle the txn's result otherwise
+        # it tries to call fetchall() on it and fails because there's no result to fetch.
+        await self.db_pool.execute(
+            "background_analyze_new_stream_ordering_column",
+            lambda txn: None,
+            "ANALYZE events(stream_ordering2)",
+        )
+
         await self.db_pool.runInteraction(
             "_background_replace_stream_ordering_column", process
         )


Test patch:
Repository: matrix-org/synapse

Issue:
Module API does not permit running looping jobs on worker processes
`ModuleApi.looping_background_call` does nothing on worker processes, which is problematic if your module needs to run any sort of background job.


Discussion:

Resolving patch:
diff --git a/synapse/module_api/__init__.py b/synapse/module_api/__init__.py
--- a/synapse/module_api/__init__.py
+++ b/synapse/module_api/__init__.py
@@ -604,10 +604,15 @@ def looping_background_call(
         msec: float,
         *args,
         desc: Optional[str] = None,
+        run_on_all_instances: bool = False,
         **kwargs,
     ):
         """Wraps a function as a background process and calls it repeatedly.
 
+        NOTE: Will only run on the instance that is configured to run
+        background processes (which is the main process by default), unless
+        `run_on_all_workers` is set.
+
         Waits `msec` initially before calling `f` for the first time.
 
         Args:
@@ -618,12 +623,14 @@ def looping_background_call(
             msec: How long to wait between calls in milliseconds.
             *args: Positional arguments to pass to function.
             desc: The background task's description. Default to the function's name.
+            run_on_all_instances: Whether to run this on all instances, rather
+                than just the instance configured to run background tasks.
             **kwargs: Key arguments to pass to function.
         """
         if desc is None:
             desc = f.__name__
 
-        if self._hs.config.run_background_tasks:
+        if self._hs.config.run_background_tasks or run_on_all_instances:
             self._clock.looping_call(
                 run_as_background_process,
                 msec,


Test patch:
Repository: matrix-org/synapse

Issue:
`txn` being redefined in for loop as we are iterating over it
Spurred on by the note in https://github.com/matrix-org/synapse/pull/10734 about `txn` being redefined in for loop as we are iterating over it.

> We were originally iterating over `txn`, which can work, but as `txn` was being used to run a
simple_delete operation inside the loop, it was actually being redefined mid-loop.
>
> https://github.com/matrix-org/synapse/pull/10734

I was curious if there were other cases in the codebase, so I searched for `in txn:` and found one more case. It looks like this would cause a problem if there was an event where the `event_auth_chains` -> `chain_id` was not defined.

https://github.com/matrix-org/synapse/blob/c586d6803a2adebcdd486be2d9eac1f62fd7d4ab/synapse/storage/databases/main/events.py#L578-L588


## Linting

Maybe we should disallow iterating over `txn` directly in favor of using `txn.fetchall()`


Discussion:
Oh, good catch. I think its unlikely for that case to ever be hit in practice as its really only used to handle out of band memberships and we likely only have one of those per room anyway. We should still fix it though!

> Maybe we should disallow iterating over `txn` directly in favor of using `txn.fetchall()`

Using `txn` can be considerably faster when there's a lot of rows to process, so I'm not super keen on forbidding it entirely. Ideally mypy would have a way to annotate that we can't reuse txn here, but I don't think its possible.

Resolving patch:
diff --git a/synapse/storage/databases/main/events.py b/synapse/storage/databases/main/events.py
--- a/synapse/storage/databases/main/events.py
+++ b/synapse/storage/databases/main/events.py
@@ -575,7 +575,13 @@ def _add_chain_cover_index(
 
             missing_auth_chains.clear()
 
-            for auth_id, event_type, state_key, chain_id, sequence_number in txn:
+            for (
+                auth_id,
+                event_type,
+                state_key,
+                chain_id,
+                sequence_number,
+            ) in txn.fetchall():
                 event_to_types[auth_id] = (event_type, state_key)
 
                 if chain_id is None:


Test patch:
Repository: matrix-org/synapse

Issue:
knock events sent over federation have no state but are not marked as outliers
when we send out a knock event, we persist the sent event in the events table, but do not mark it out as an outlier.

We give it a state_group, but that state_group contains only the knock event itself.


Discussion:
I'm not entirely sure what the effects of this will be, but it certainly isn't a good thing for the consistency of the room state and the DAG.

Resolving patch:
diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -593,6 +593,13 @@ async def do_knock(
             target_hosts, room_id, knockee, Membership.KNOCK, content, params=params
         )
 
+        # Mark the knock as an outlier as we don't yet have the state at this point in
+        # the DAG.
+        event.internal_metadata.outlier = True
+
+        # ... but tell /sync to send it to clients anyway.
+        event.internal_metadata.out_of_band_membership = True
+
         # Record the room ID and its version so that we have a record of the room
         await self._maybe_store_room_on_outlier_membership(
             room_id=event.room_id, room_version=event_format_version


Test patch:
Repository: matrix-org/synapse

Issue:
Recent releases are not signed
It seems that [recent releases](https://github.com/matrix-org/synapse/releases) of Synapse are not being signed any longer. I think this is due to the [release script](https://github.com/matrix-org/synapse/blob/14b8c0476f93ea2ed3134e75733e45aa0ab6f5a5/scripts-dev/release.py) not properly signing the tags, but I'm unsure.

(Note that @anoadragon453 mentioned he has been doing the tags manually so his releases do have signatures on them.)

@erikjohnston mentioned this might be due to already having the necessary keys unlocked or not.


Discussion:
We probably need to add an option to: https://github.com/matrix-org/synapse/blob/6878e1065308caf0f79e380b4de1433ab1487a34/scripts-dev/release.py#L279

The docs for the ~~github lib is at https://pygithub.readthedocs.io/en/latest/reference.html~~ git lib is at https://gitpython.readthedocs.io/en/stable/

Resolving patch:
diff --git a/scripts-dev/release.py b/scripts-dev/release.py
--- a/scripts-dev/release.py
+++ b/scripts-dev/release.py
@@ -276,7 +276,7 @@ def tag(gh_token: Optional[str]):
     if click.confirm("Edit text?", default=False):
         changes = click.edit(changes, require_save=False)
 
-    repo.create_tag(tag_name, message=changes)
+    repo.create_tag(tag_name, message=changes, sign=True)
 
     if not click.confirm("Push tag to GitHub?", default=True):
         print("")


Test patch:
Repository: matrix-org/synapse

Issue:
Exceptions from presence handling in 1.44.0rc1
```
2021-09-29 16:09:28,131 - synapse.metrics.background_process_metrics - 215 - ERROR - notify_interested_services_ephemeral-0 - Background process 'notify_interested_services_ephemeral' threw an exception
Traceback (most recent call last):
  File "synapse/metrics/background_process_metrics.py", line 213, in run
    return await maybe_awaitable(func(*args, **kwargs))
  File "synapse/handlers/appservice.py", line 247, in _notify_interested_services_ephemeral
    events = await self._handle_presence(service, users)
  File "synapse/handlers/appservice.py", line 293, in _handle_presence
    presence_events, _ = await presence_source.get_new_events(
  File "synapse/logging/utils.py", line 69, in wrapped
    return f(*args, **kwargs)
TypeError: get_new_events() got an unexpected keyword argument 'service'
```

looks like something changed here in 4054dfa409fa17b45ab8f265813994956ed97bae


Discussion:

Resolving patch:
diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py
--- a/synapse/handlers/presence.py
+++ b/synapse/handlers/presence.py
@@ -52,6 +52,7 @@
 from synapse.api.constants import EventTypes, Membership, PresenceState
 from synapse.api.errors import SynapseError
 from synapse.api.presence import UserPresenceState
+from synapse.appservice import ApplicationService
 from synapse.events.presence_router import PresenceRouter
 from synapse.logging.context import run_in_background
 from synapse.logging.utils import log_function
@@ -1521,10 +1522,11 @@ async def get_new_events(
         user: UserID,
         from_key: Optional[int],
         limit: Optional[int] = None,
-        room_ids: Optional[List[str]] = None,
+        room_ids: Optional[Collection[str]] = None,
         is_guest: bool = False,
         explicit_room_id: Optional[str] = None,
         include_offline: bool = True,
+        service: Optional[ApplicationService] = None,
     ) -> Tuple[List[UserPresenceState], int]:
         # The process for getting presence events are:
         #  1. Get the rooms the user is in.


Test patch:
Repository: matrix-org/synapse

Issue:
test_export_data_command.sh CI script hung
I had an instance of the [test_export_data_command](https://github.com/matrix-org/synapse/blob/c7a5e49664ab0bd18a57336e282fa6c3b9a17ca0/.ci/scripts/test_export_data_command.sh) script running for 6h in CI before eventually being timed out: https://github.com/matrix-org/synapse/runs/4010916318?check_suite_focus=true

It looks like the script got to this line before simply stopping:

https://github.com/matrix-org/synapse/blob/c7a5e49664ab0bd18a57336e282fa6c3b9a17ca0/.ci/scripts/test_export_data_command.sh#L24

Perhaps a case of Synapse failing to shut down properly?

Should we place a smaller timeout on (a part of) this script - which would fail the test, but without burning so much CI time - in the short term?


Discussion:

Resolving patch:
diff --git a/synapse/app/_base.py b/synapse/app/_base.py
--- a/synapse/app/_base.py
+++ b/synapse/app/_base.py
@@ -346,6 +346,7 @@ async def start(hs: "HomeServer"):
     # numbers of DNS requests don't starve out other users of the threadpool.
     resolver_threadpool = ThreadPool(name="gai_resolver")
     resolver_threadpool.start()
+    reactor.addSystemEventTrigger("during", "shutdown", resolver_threadpool.stop)
     reactor.installNameResolver(
         GAIResolver(reactor, getThreadPool=lambda: resolver_threadpool)
     )


Test patch:
Repository: matrix-org/synapse

Issue:
remove unused `room_stats_historical` and `user_stats_historical` tables
since #9721, these tables are no longer used. Once we no longer need to support rollback to Synapse 1.38, we should add database schema deltas to drop the tables altogether.


Discussion:

Resolving patch:
diff --git a/synapse/storage/schema/__init__.py b/synapse/storage/schema/__init__.py
--- a/synapse/storage/schema/__init__.py
+++ b/synapse/storage/schema/__init__.py
@@ -45,10 +45,13 @@
 Changes in SCHEMA_VERSION = 65:
     - MSC2716: Remove unique event_id constraint from insertion_event_edges
       because an insertion event can have multiple edges.
+    - Remove unused tables `user_stats_historical` and `room_stats_historical`.
 """
 
 
-SCHEMA_COMPAT_VERSION = 60  # 60: "outlier" not in internal_metadata.
+SCHEMA_COMPAT_VERSION = (
+    61  # 61: Remove unused tables `user_stats_historical` and `room_stats_historical`
+)
 """Limit on how far the synapse codebase can be rolled back without breaking db compat
 
 This value is stored in the database, and checked on startup. If the value in the


Test patch:
Repository: matrix-org/synapse

Issue:
Stop Synapse master process from starting if `worker_app` is set
It's easy to mess up and try and start Synapse via `synapse.app.homeserver` with config that contains `worker_app` (e.g. the Docker image by default uses `synapse.app.homeserver`)


Discussion:

Resolving patch:
diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -357,6 +357,13 @@ def setup(config_options: List[str]) -> SynapseHomeServer:
         # generating config files and shouldn't try to continue.
         sys.exit(0)
 
+    if config.worker.worker_app:
+        raise ConfigError(
+            "You have specified `worker_app` in the config but are attempting to start a non-worker "
+            "instance. Please use `python -m synapse.app.generic_worker` instead (or remove the option if this is the main process)."
+        )
+        sys.exit(1)
+
     events.USE_FROZEN_DICTS = config.server.use_frozen_dicts
     synapse.util.caches.TRACK_MEMORY_USAGE = config.caches.track_memory_usage
 


Test patch:
diff --git a/tests/app/test_homeserver_start.py b/tests/app/test_homeserver_start.py
new file mode 100644
--- /dev/null
+++ b/tests/app/test_homeserver_start.py
@@ -0,0 +1,31 @@
+# Copyright 2021 The Matrix.org Foundation C.I.C.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import synapse.app.homeserver
+from synapse.config._base import ConfigError
+
+from tests.config.utils import ConfigFileTestCase
+
+
+class HomeserverAppStartTestCase(ConfigFileTestCase):
+    def test_wrong_start_caught(self):
+        # Generate a config with a worker_app
+        self.generate_config()
+        # Add a blank line as otherwise the next addition ends up on a line with a comment
+        self.add_lines_to_config(["  "])
+        self.add_lines_to_config(["worker_app: test_worker_app"])
+
+        # Ensure that starting master process with worker config raises an exception
+        with self.assertRaises(ConfigError):
+            synapse.app.homeserver.setup(["-c", self.config_file])
Repository: matrix-org/synapse

Issue:
Error on shutdown when prescence is enabled
Spotted when I was reading through sytest logs.

```
21082-2021-12-06 18:04:19,715 - synapse.metrics.background_process_metrics - 246 - ERROR - generic_presence.on_shutdown-0 - Background process 'generic_presence.on_shutdown' threw an exception
21083-Traceback (most recent call last):
21084-  File "/venv/lib/python3.7/site-packages/synapse/metrics/background_process_metrics.py", line 242, in run
21085-    return await func(*args, **kwargs)
21086-TypeError: object NoneType can't be used in 'await' expression
```

The problem is that we register a call to `run_as_background_process` before shutdown here. We pass it the callable `self._on_shutdown`. The latter is a synchronous function, but `run_as_background_process` expects an async function.

https://github.com/matrix-org/synapse/blob/75ca0a6168f92dab3255839cf85fb0df3a0076c3/synapse/handlers/presence.py#L416-L428

The symptom was introduced in #10847 when we removed `maybe_awaitable`. But the underlying cause is that `_on_shutdown` didn't return a Deferred back when we expected it to. (Plus `addSystemEventTrigger` obscures the call from mypy. Probably extra confusion when this was rewritten from twisted defers to asyncs.)


Discussion:

Resolving patch:
diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py
--- a/synapse/handlers/presence.py
+++ b/synapse/handlers/presence.py
@@ -421,7 +421,7 @@ def __init__(self, hs: "HomeServer"):
             self._on_shutdown,
         )
 
-    def _on_shutdown(self) -> None:
+    async def _on_shutdown(self) -> None:
         if self._presence_enabled:
             self.hs.get_tcp_replication().send_command(
                 ClearUserSyncsCommand(self.instance_id)


Test patch: