Repository: matrix-org/synapse

Issue:
Experimental join_rule for joining rooms in a space without an invite
For the Spaces project there needs to be a way to allow users to join a room based on membership in a space.

An approach for this is detailed in [MSC3083](https://github.com/matrix-org/matrix-doc/pull/3083).

The approach will be to:
* Create an experimental room version.
* Update the auth rules based on the `join_rule` / experimental room version to allow users to join based on the `allowed` key in the `m.room.join_rules` event.

Note that we want this to be in an experimental room version as it is unclear whether clients will need to make changes to support this.

Discussion:

Patch:
diff --git a/changelog.d/9685.misc b/changelog.d/9685.misc
new file mode 100644
index 000000000..0506d8af0
--- /dev/null
+++ b/changelog.d/9685.misc
@@ -0,0 +1,1 @@
+Update `scripts-dev/complement.sh` to use a local checkout of Complement, allow running a subset of tests and have it use Synapse's Complement test blacklist.
\ No newline at end of file
diff --git a/changelog.d/9717.feature b/changelog.d/9717.feature
new file mode 100644
index 000000000..c2c74f13d
--- /dev/null
+++ b/changelog.d/9717.feature
@@ -0,0 +1,1 @@
+Add experimental support for [MSC3083](https://github.com/matrix-org/matrix-doc/pull/3083): restricting room access via group membership.
diff --git a/changelog.d/9718.removal b/changelog.d/9718.removal
new file mode 100644
index 000000000..6de781421
--- /dev/null
+++ b/changelog.d/9718.removal
@@ -0,0 +1,1 @@
+Replace deprecated `imp` module with successor `importlib`. Contributed by Cristina Muñoz.
diff --git a/changelog.d/9719.doc b/changelog.d/9719.doc
new file mode 100644
index 000000000..f018606dd
--- /dev/null
+++ b/changelog.d/9719.doc
@@ -0,0 +1,1 @@
+Make the allowed_local_3pids regex example in the sample config stricter.
diff --git a/changelog.d/9720.misc b/changelog.d/9720.misc
new file mode 100644
index 000000000..9dd0bc9cf
--- /dev/null
+++ b/changelog.d/9720.misc
@@ -0,0 +1,1 @@
+Revert using `dmypy run` in lint script.
diff --git a/docs/sample_config.yaml b/docs/sample_config.yaml
index c73ea6b16..b0bf98774 100644
--- a/docs/sample_config.yaml
+++ b/docs/sample_config.yaml
@@ -1246,9 +1246,9 @@ account_validity:
 #
 #allowed_local_3pids:
 #  - medium: email
-#    pattern: '.*@matrix\.org'
+#    pattern: '^[^@]+@matrix\.org$'
 #  - medium: email
-#    pattern: '.*@vector\.im'
+#    pattern: '^[^@]+@vector\.im$'
 #  - medium: msisdn
 #    pattern: '\+44'
 
diff --git a/scripts-dev/complement.sh b/scripts-dev/complement.sh
index 3cde53f5c..31cc20a82 100755
--- a/scripts-dev/complement.sh
+++ b/scripts-dev/complement.sh
@@ -1,22 +1,49 @@
-#! /bin/bash -eu
+#!/usr/bin/env bash
 # This script is designed for developers who want to test their code
 # against Complement.
 #
 # It makes a Synapse image which represents the current checkout,
-# then downloads Complement and runs it with that image.
+# builds a synapse-complement image on top, then runs tests with it.
+#
+# By default the script will fetch the latest Complement master branch and
+# run tests with that. This can be overridden to use a custom Complement
+# checkout by setting the COMPLEMENT_DIR environment variable to the
+# filepath of a local Complement checkout.
+#
+# A regular expression of test method names can be supplied as the first
+# argument to the script. Complement will then only run those tests. If
+# no regex is supplied, all tests are run. For example;
+#
+# ./complement.sh "TestOutboundFederation(Profile|Send)"
+#
+
+# Exit if a line returns a non-zero exit code
+set -e
 
+# Change to the repository root
 cd "$(dirname $0)/.."
 
+# Check for a user-specified Complement checkout
+if [[ -z "$COMPLEMENT_DIR" ]]; then
+  echo "COMPLEMENT_DIR not set. Fetching the latest Complement checkout..."
+  wget -Nq https://github.com/matrix-org/complement/archive/master.tar.gz
+  tar -xzf master.tar.gz
+  COMPLEMENT_DIR=complement-master
+  echo "Checkout available at 'complement-master'"
+fi
+
 # Build the base Synapse image from the local checkout
-docker build -t matrixdotorg/synapse:latest -f docker/Dockerfile .
+docker build -t matrixdotorg/synapse -f docker/Dockerfile .
+# Build the Synapse monolith image from Complement, based on the above image we just built
+docker build -t complement-synapse -f "$COMPLEMENT_DIR/dockerfiles/Synapse.Dockerfile" "$COMPLEMENT_DIR/dockerfiles"
 
-# Download Complement
-wget -N https://github.com/matrix-org/complement/archive/master.tar.gz
-tar -xzf master.tar.gz
-cd complement-master
+cd "$COMPLEMENT_DIR"
 
-# Build the Synapse image from Complement, based on the above image we just built
-docker build -t complement-synapse -f dockerfiles/Synapse.Dockerfile ./dockerfiles
+EXTRA_COMPLEMENT_ARGS=""
+if [[ -n "$1" ]]; then
+  # A test name regex has been set, supply it to Complement
+  EXTRA_COMPLEMENT_ARGS+="-run $1 "
+fi
 
-# Run the tests on the resulting image!
-COMPLEMENT_BASE_IMAGE=complement-synapse go test -v -count=1 ./tests
+# Run the tests!
+COMPLEMENT_BASE_IMAGE=complement-synapse go test -v -tags synapse_blacklist -count=1 $EXTRA_COMPLEMENT_ARGS ./tests
diff --git a/scripts-dev/lint.sh b/scripts-dev/lint.sh
index 41415ee07..9761e9759 100755
--- a/scripts-dev/lint.sh
+++ b/scripts-dev/lint.sh
@@ -95,4 +95,4 @@ isort "${files[@]}"
 python3 -m black "${files[@]}"
 ./scripts-dev/config-lint.sh
 flake8 "${files[@]}"
-dmypy run
+mypy
diff --git a/synapse/api/constants.py b/synapse/api/constants.py
index 8f37d2cf3..6856dab06 100644
--- a/synapse/api/constants.py
+++ b/synapse/api/constants.py
@@ -59,6 +59,8 @@ class JoinRules:
     KNOCK = "knock"
     INVITE = "invite"
     PRIVATE = "private"
+    # As defined for MSC3083.
+    MSC3083_RESTRICTED = "restricted"
 
 
 class LoginType:
diff --git a/synapse/api/room_versions.py b/synapse/api/room_versions.py
index de2cc15d3..87038d436 100644
--- a/synapse/api/room_versions.py
+++ b/synapse/api/room_versions.py
@@ -57,7 +57,7 @@ class RoomVersion:
     state_res = attr.ib(type=int)  # one of the StateResolutionVersions
     enforce_key_validity = attr.ib(type=bool)
 
-    # bool: before MSC2261/MSC2432, m.room.aliases had special auth rules and redaction rules
+    # Before MSC2261/MSC2432, m.room.aliases had special auth rules and redaction rules
     special_case_aliases_auth = attr.ib(type=bool)
     # Strictly enforce canonicaljson, do not allow:
     # * Integers outside the range of [-2 ^ 53 + 1, 2 ^ 53 - 1]
@@ -69,6 +69,8 @@ class RoomVersion:
     limit_notifications_power_levels = attr.ib(type=bool)
     # MSC2174/MSC2176: Apply updated redaction rules algorithm.
     msc2176_redaction_rules = attr.ib(type=bool)
+    # MSC3083: Support the 'restricted' join_rule.
+    msc3083_join_rules = attr.ib(type=bool)
 
 
 class RoomVersions:
@@ -82,6 +84,7 @@ class RoomVersions:
         strict_canonicaljson=False,
         limit_notifications_power_levels=False,
         msc2176_redaction_rules=False,
+        msc3083_join_rules=False,
     )
     V2 = RoomVersion(
         "2",
@@ -93,6 +96,7 @@ class RoomVersions:
         strict_canonicaljson=False,
         limit_notifications_power_levels=False,
         msc2176_redaction_rules=False,
+        msc3083_join_rules=False,
     )
     V3 = RoomVersion(
         "3",
@@ -104,6 +108,7 @@ class RoomVersions:
         strict_canonicaljson=False,
         limit_notifications_power_levels=False,
         msc2176_redaction_rules=False,
+        msc3083_join_rules=False,
     )
     V4 = RoomVersion(
         "4",
@@ -115,6 +120,7 @@ class RoomVersions:
         strict_canonicaljson=False,
         limit_notifications_power_levels=False,
         msc2176_redaction_rules=False,
+        msc3083_join_rules=False,
     )
     V5 = RoomVersion(
         "5",
@@ -126,6 +132,7 @@ class RoomVersions:
         strict_canonicaljson=False,
         limit_notifications_power_levels=False,
         msc2176_redaction_rules=False,
+        msc3083_join_rules=False,
     )
     V6 = RoomVersion(
         "6",
@@ -137,6 +144,7 @@ class RoomVersions:
         strict_canonicaljson=True,
         limit_notifications_power_levels=True,
         msc2176_redaction_rules=False,
+        msc3083_join_rules=False,
     )
     MSC2176 = RoomVersion(
         "org.matrix.msc2176",
@@ -148,6 +156,19 @@ class RoomVersions:
         strict_canonicaljson=True,
         limit_notifications_power_levels=True,
         msc2176_redaction_rules=True,
+        msc3083_join_rules=False,
+    )
+    MSC3083 = RoomVersion(
+        "org.matrix.msc3083",
+        RoomDisposition.UNSTABLE,
+        EventFormatVersions.V3,
+        StateResolutionVersions.V2,
+        enforce_key_validity=True,
+        special_case_aliases_auth=False,
+        strict_canonicaljson=True,
+        limit_notifications_power_levels=True,
+        msc2176_redaction_rules=False,
+        msc3083_join_rules=True,
     )
 
 
@@ -162,4 +183,5 @@ KNOWN_ROOM_VERSIONS = {
         RoomVersions.V6,
         RoomVersions.MSC2176,
     )
+    # Note that we do not include MSC3083 here unless it is enabled in the config.
 }  # type: Dict[str, RoomVersion]
diff --git a/synapse/config/experimental.py b/synapse/config/experimental.py
index 86f4d9af9..eb96ecda7 100644
--- a/synapse/config/experimental.py
+++ b/synapse/config/experimental.py
@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from synapse.api.room_versions import KNOWN_ROOM_VERSIONS, RoomVersions
 from synapse.config._base import Config
 from synapse.types import JsonDict
 
@@ -27,7 +28,11 @@ class ExperimentalConfig(Config):
 
         # MSC2858 (multiple SSO identity providers)
         self.msc2858_enabled = experimental.get("msc2858_enabled", False)  # type: bool
-        # Spaces (MSC1772, MSC2946, etc)
+
+        # Spaces (MSC1772, MSC2946, MSC3083, etc)
         self.spaces_enabled = experimental.get("spaces_enabled", False)  # type: bool
+        if self.spaces_enabled:
+            KNOWN_ROOM_VERSIONS[RoomVersions.MSC3083.identifier] = RoomVersions.MSC3083
+
         # MSC3026 (busy presence state)
         self.msc3026_enabled = experimental.get("msc3026_enabled", False)  # type: bool
diff --git a/synapse/config/registration.py b/synapse/config/registration.py
index ead007ba5..f27d1e14a 100644
--- a/synapse/config/registration.py
+++ b/synapse/config/registration.py
@@ -298,9 +298,9 @@ class RegistrationConfig(Config):
         #
         #allowed_local_3pids:
         #  - medium: email
-        #    pattern: '.*@matrix\\.org'
+        #    pattern: '^[^@]+@matrix\\.org$'
         #  - medium: email
-        #    pattern: '.*@vector\\.im'
+        #    pattern: '^[^@]+@vector\\.im$'
         #  - medium: msisdn
         #    pattern: '\\+44'
 
diff --git a/synapse/event_auth.py b/synapse/event_auth.py
index 91ad5b3d3..9863953f5 100644
--- a/synapse/event_auth.py
+++ b/synapse/event_auth.py
@@ -162,7 +162,7 @@ def check(
         logger.debug("Auth events: %s", [a.event_id for a in auth_events.values()])
 
     if event.type == EventTypes.Member:
-        _is_membership_change_allowed(event, auth_events)
+        _is_membership_change_allowed(room_version_obj, event, auth_events)
         logger.debug("Allowing! %s", event)
         return
 
@@ -220,8 +220,19 @@ def _can_federate(event: EventBase, auth_events: StateMap[EventBase]) -> bool:
 
 
 def _is_membership_change_allowed(
-    event: EventBase, auth_events: StateMap[EventBase]
+    room_version: RoomVersion, event: EventBase, auth_events: StateMap[EventBase]
 ) -> None:
+    """
+    Confirms that the event which changes membership is an allowed change.
+
+    Args:
+        room_version: The version of the room.
+        event: The event to check.
+        auth_events: The current auth events of the room.
+
+    Raises:
+        AuthError if the event is not allowed.
+    """
     membership = event.content["membership"]
 
     # Check if this is the room creator joining:
@@ -315,14 +326,19 @@ def _is_membership_change_allowed(
             if user_level < invite_level:
                 raise AuthError(403, "You don't have permission to invite users")
     elif Membership.JOIN == membership:
-        # Joins are valid iff caller == target and they were:
-        # invited: They are accepting the invitation
-        # joined: It's a NOOP
+        # Joins are valid iff caller == target and:
+        # * They are not banned.
+        # * They are accepting a previously sent invitation.
+        # * They are already joined (it's a NOOP).
+        # * The room is public or restricted.
         if event.user_id != target_user_id:
             raise AuthError(403, "Cannot force another user to join.")
         elif target_banned:
             raise AuthError(403, "You are banned from this room")
-        elif join_rule == JoinRules.PUBLIC:
+        elif join_rule == JoinRules.PUBLIC or (
+            room_version.msc3083_join_rules
+            and join_rule == JoinRules.MSC3083_RESTRICTED
+        ):
             pass
         elif join_rule == JoinRules.INVITE:
             if not caller_in_room and not caller_invited:
diff --git a/synapse/storage/prepare_database.py b/synapse/storage/prepare_database.py
index 6c3c2da52..c7f0b8ccb 100644
--- a/synapse/storage/prepare_database.py
+++ b/synapse/storage/prepare_database.py
@@ -13,7 +13,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-import imp
+import importlib.util
 import logging
 import os
 import re
@@ -454,8 +454,13 @@ def _upgrade_existing_database(
                     )
 
                 module_name = "synapse.storage.v%d_%s" % (v, root_name)
-                with open(absolute_path) as python_file:
-                    module = imp.load_source(module_name, absolute_path, python_file)  # type: ignore
+
+                spec = importlib.util.spec_from_file_location(
+                    module_name, absolute_path
+                )
+                module = importlib.util.module_from_spec(spec)
+                spec.loader.exec_module(module)  # type: ignore
+
                 logger.info("Running script %s", relative_path)
                 module.run_create(cur, database_engine)  # type: ignore
                 if not is_empty:


Test patch:
diff --git a/tests/test_event_auth.py b/tests/test_event_auth.py
index 3f2691ee6..b5f18344d 100644
--- a/tests/test_event_auth.py
+++ b/tests/test_event_auth.py
@@ -207,6 +207,226 @@ class EventAuthTestCase(unittest.TestCase):
                 do_sig_check=False,
             )
 
+    def test_join_rules_public(self):
+        """
+        Test joining a public room.
+        """
+        creator = "@creator:example.com"
+        pleb = "@joiner:example.com"
+
+        auth_events = {
+            ("m.room.create", ""): _create_event(creator),
+            ("m.room.member", creator): _join_event(creator),
+            ("m.room.join_rules", ""): _join_rules_event(creator, "public"),
+        }
+
+        # Check join.
+        event_auth.check(
+            RoomVersions.V6,
+            _join_event(pleb),
+            auth_events,
+            do_sig_check=False,
+        )
+
+        # A user cannot be force-joined to a room.
+        with self.assertRaises(AuthError):
+            event_auth.check(
+                RoomVersions.V6,
+                _member_event(pleb, "join", sender=creator),
+                auth_events,
+                do_sig_check=False,
+            )
+
+        # Banned should be rejected.
+        auth_events[("m.room.member", pleb)] = _member_event(pleb, "ban")
+        with self.assertRaises(AuthError):
+            event_auth.check(
+                RoomVersions.V6,
+                _join_event(pleb),
+                auth_events,
+                do_sig_check=False,
+            )
+
+        # A user who left can re-join.
+        auth_events[("m.room.member", pleb)] = _member_event(pleb, "leave")
+        event_auth.check(
+            RoomVersions.V6,
+            _join_event(pleb),
+            auth_events,
+            do_sig_check=False,
+        )
+
+        # A user can send a join if they're in the room.
+        auth_events[("m.room.member", pleb)] = _member_event(pleb, "join")
+        event_auth.check(
+            RoomVersions.V6,
+            _join_event(pleb),
+            auth_events,
+            do_sig_check=False,
+        )
+
+        # A user can accept an invite.
+        auth_events[("m.room.member", pleb)] = _member_event(
+            pleb, "invite", sender=creator
+        )
+        event_auth.check(
+            RoomVersions.V6,
+            _join_event(pleb),
+            auth_events,
+            do_sig_check=False,
+        )
+
+    def test_join_rules_invite(self):
+        """
+        Test joining an invite only room.
+        """
+        creator = "@creator:example.com"
+        pleb = "@joiner:example.com"
+
+        auth_events = {
+            ("m.room.create", ""): _create_event(creator),
+            ("m.room.member", creator): _join_event(creator),
+            ("m.room.join_rules", ""): _join_rules_event(creator, "invite"),
+        }
+
+        # A join without an invite is rejected.
+        with self.assertRaises(AuthError):
+            event_auth.check(
+                RoomVersions.V6,
+                _join_event(pleb),
+                auth_events,
+                do_sig_check=False,
+            )
+
+        # A user cannot be force-joined to a room.
+        with self.assertRaises(AuthError):
+            event_auth.check(
+                RoomVersions.V6,
+                _member_event(pleb, "join", sender=creator),
+                auth_events,
+                do_sig_check=False,
+            )
+
+        # Banned should be rejected.
+        auth_events[("m.room.member", pleb)] = _member_event(pleb, "ban")
+        with self.assertRaises(AuthError):
+            event_auth.check(
+                RoomVersions.V6,
+                _join_event(pleb),
+                auth_events,
+                do_sig_check=False,
+            )
+
+        # A user who left cannot re-join.
+        auth_events[("m.room.member", pleb)] = _member_event(pleb, "leave")
+        with self.assertRaises(AuthError):
+            event_auth.check(
+                RoomVersions.V6,
+                _join_event(pleb),
+                auth_events,
+                do_sig_check=False,
+            )
+
+        # A user can send a join if they're in the room.
+        auth_events[("m.room.member", pleb)] = _member_event(pleb, "join")
+        event_auth.check(
+            RoomVersions.V6,
+            _join_event(pleb),
+            auth_events,
+            do_sig_check=False,
+        )
+
+        # A user can accept an invite.
+        auth_events[("m.room.member", pleb)] = _member_event(
+            pleb, "invite", sender=creator
+        )
+        event_auth.check(
+            RoomVersions.V6,
+            _join_event(pleb),
+            auth_events,
+            do_sig_check=False,
+        )
+
+    def test_join_rules_msc3083_restricted(self):
+        """
+        Test joining a restricted room from MSC3083.
+
+        This is pretty much the same test as public.
+        """
+        creator = "@creator:example.com"
+        pleb = "@joiner:example.com"
+
+        auth_events = {
+            ("m.room.create", ""): _create_event(creator),
+            ("m.room.member", creator): _join_event(creator),
+            ("m.room.join_rules", ""): _join_rules_event(creator, "restricted"),
+        }
+
+        # Older room versions don't understand this join rule
+        with self.assertRaises(AuthError):
+            event_auth.check(
+                RoomVersions.V6,
+                _join_event(pleb),
+                auth_events,
+                do_sig_check=False,
+            )
+
+        # Check join.
+        event_auth.check(
+            RoomVersions.MSC3083,
+            _join_event(pleb),
+            auth_events,
+            do_sig_check=False,
+        )
+
+        # A user cannot be force-joined to a room.
+        with self.assertRaises(AuthError):
+            event_auth.check(
+                RoomVersions.MSC3083,
+                _member_event(pleb, "join", sender=creator),
+                auth_events,
+                do_sig_check=False,
+            )
+
+        # Banned should be rejected.
+        auth_events[("m.room.member", pleb)] = _member_event(pleb, "ban")
+        with self.assertRaises(AuthError):
+            event_auth.check(
+                RoomVersions.MSC3083,
+                _join_event(pleb),
+                auth_events,
+                do_sig_check=False,
+            )
+
+        # A user who left can re-join.
+        auth_events[("m.room.member", pleb)] = _member_event(pleb, "leave")
+        event_auth.check(
+            RoomVersions.MSC3083,
+            _join_event(pleb),
+            auth_events,
+            do_sig_check=False,
+        )
+
+        # A user can send a join if they're in the room.
+        auth_events[("m.room.member", pleb)] = _member_event(pleb, "join")
+        event_auth.check(
+            RoomVersions.MSC3083,
+            _join_event(pleb),
+            auth_events,
+            do_sig_check=False,
+        )
+
+        # A user can accept an invite.
+        auth_events[("m.room.member", pleb)] = _member_event(
+            pleb, "invite", sender=creator
+        )
+        event_auth.check(
+            RoomVersions.MSC3083,
+            _join_event(pleb),
+            auth_events,
+            do_sig_check=False,
+        )
+
 
 # helpers for making events
 
@@ -225,19 +445,24 @@ def _create_event(user_id):
     )
 
 
-def _join_event(user_id):
+def _member_event(user_id, membership, sender=None):
     return make_event_from_dict(
         {
             "room_id": TEST_ROOM_ID,
             "event_id": _get_event_id(),
             "type": "m.room.member",
-            "sender": user_id,
+            "sender": sender or user_id,
             "state_key": user_id,
-            "content": {"membership": "join"},
+            "content": {"membership": membership},
+            "prev_events": [],
         }
     )
 
 
+def _join_event(user_id):
+    return _member_event(user_id, "join")
+
+
 def _power_levels_event(sender, content):
     return make_event_from_dict(
         {
@@ -277,6 +502,21 @@ def _random_state_event(sender):
     )
 
 
+def _join_rules_event(sender, join_rule):
+    return make_event_from_dict(
+        {
+            "room_id": TEST_ROOM_ID,
+            "event_id": _get_event_id(),
+            "type": "m.room.join_rules",
+            "sender": sender,
+            "state_key": "",
+            "content": {
+                "join_rule": join_rule,
+            },
+        }
+    )
+
+
 event_count = 0