Repository: matrix-org/synapse

Issue:
Errors from the manhole go to the logfile rather than the manhole
This is only happening on matrix.org, and I'm sure it didn't previously:

in the manhole:
```
>>> slkghdklg
>>> 
```

meanwhile, in the log:
```
2019-02-28 17:14:03,867 - twisted - 240 - ERROR -  - Traceback (most recent call last):
2019-02-28 17:14:03,884 - twisted - 240 - ERROR -  -   File /usr/local/lib/python3.6/code.py, line 91, in runcode
2019-02-28 17:14:03,896 - twisted - 240 - ERROR -  -     exec(code, self.locals)
2019-02-28 17:14:03,908 - twisted - 240 - ERROR -  -   File <console>, line 1, in <module>
2019-02-28 17:14:03,917 - twisted - 240 - ERROR -  - NameError: name 'slkghdklg' is not defined
```


Discussion:
erikjohnston: How was sentry intercepting this? I assume its not logging as sentry would just pass that on

Patch:
diff --git a/changelog.d/5027.feature b/changelog.d/5027.feature
new file mode 100644
index 000000000..12766a82a
--- /dev/null
+++ b/changelog.d/5027.feature
@@ -0,0 +1,1 @@
+Add time-based account expiration.
diff --git a/changelog.d/5035.bugfix b/changelog.d/5035.bugfix
new file mode 100644
index 000000000..85e154027
--- /dev/null
+++ b/changelog.d/5035.bugfix
@@ -0,0 +1,1 @@
+Fix disappearing exceptions in manhole.
diff --git a/docs/sample_config.yaml b/docs/sample_config.yaml
index a380b5f9a..8888fd49c 100644
--- a/docs/sample_config.yaml
+++ b/docs/sample_config.yaml
@@ -646,6 +646,12 @@ uploads_path: "DATADIR/uploads"
 #
 #enable_registration: false
 
+# Optional account validity parameter. This allows for, e.g., accounts to
+# be denied any request after a given period.
+#
+#account_validity:
+#  period: 6w
+
 # The user must provide all of the below types of 3PID when registering.
 #
 #registrations_require_3pid:
diff --git a/synapse/api/auth.py b/synapse/api/auth.py
index e8112d5f0..976e0dd18 100644
--- a/synapse/api/auth.py
+++ b/synapse/api/auth.py
@@ -64,6 +64,8 @@ class Auth(object):
         self.token_cache = LruCache(CACHE_SIZE_FACTOR * 10000)
         register_cache("cache", "token_cache", self.token_cache)
 
+        self._account_validity = hs.config.account_validity
+
     @defer.inlineCallbacks
     def check_from_context(self, room_version, event, context, do_sig_check=True):
         prev_state_ids = yield context.get_prev_state_ids(self.store)
@@ -226,6 +228,16 @@ class Auth(object):
             token_id = user_info["token_id"]
             is_guest = user_info["is_guest"]
 
+            # Deny the request if the user account has expired.
+            if self._account_validity.enabled:
+                expiration_ts = yield self.store.get_expiration_ts_for_user(user)
+                if self.clock.time_msec() >= expiration_ts:
+                    raise AuthError(
+                        403,
+                        "User account has expired",
+                        errcode=Codes.EXPIRED_ACCOUNT,
+                    )
+
             # device_id may not be present if get_user_by_access_token has been
             # stubbed out.
             device_id = user_info.get("device_id")
diff --git a/synapse/api/errors.py b/synapse/api/errors.py
index 0b464834c..4c33450e7 100644
--- a/synapse/api/errors.py
+++ b/synapse/api/errors.py
@@ -60,6 +60,7 @@ class Codes(object):
     UNSUPPORTED_ROOM_VERSION = "M_UNSUPPORTED_ROOM_VERSION"
     INCOMPATIBLE_ROOM_VERSION = "M_INCOMPATIBLE_ROOM_VERSION"
     WRONG_ROOM_KEYS_VERSION = "M_WRONG_ROOM_KEYS_VERSION"
+    EXPIRED_ACCOUNT = "ORG_MATRIX_EXPIRED_ACCOUNT"
 
 
 class CodeMessageException(RuntimeError):
diff --git a/synapse/config/registration.py b/synapse/config/registration.py
index fcfda341e..dd242b121 100644
--- a/synapse/config/registration.py
+++ b/synapse/config/registration.py
@@ -20,6 +20,15 @@ from synapse.types import RoomAlias
 from synapse.util.stringutils import random_string_with_symbols
 
 
+class AccountValidityConfig(Config):
+    def __init__(self, config):
+        self.enabled = (len(config) > 0)
+
+        period = config.get("period", None)
+        if period:
+            self.period = self.parse_duration(period)
+
+
 class RegistrationConfig(Config):
 
     def read_config(self, config):
@@ -31,6 +40,8 @@ class RegistrationConfig(Config):
                 strtobool(str(config["disable_registration"]))
             )
 
+        self.account_validity = AccountValidityConfig(config.get("account_validity", {}))
+
         self.registrations_require_3pid = config.get("registrations_require_3pid", [])
         self.allowed_local_3pids = config.get("allowed_local_3pids", [])
         self.enable_3pid_lookup = config.get("enable_3pid_lookup", True)
@@ -76,6 +87,12 @@ class RegistrationConfig(Config):
         #
         #enable_registration: false
 
+        # Optional account validity parameter. This allows for, e.g., accounts to
+        # be denied any request after a given period.
+        #
+        #account_validity:
+        #  period: 6w
+
         # The user must provide all of the below types of 3PID when registering.
         #
         #registrations_require_3pid:
diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py
index e30b86c34..643f7a380 100644
--- a/synapse/storage/registration.py
+++ b/synapse/storage/registration.py
@@ -86,6 +86,26 @@ class RegistrationWorkerStore(SQLBaseStore):
             "get_user_by_access_token", self._query_for_auth, token
         )
 
+    @cachedInlineCallbacks()
+    def get_expiration_ts_for_user(self, user):
+        """Get the expiration timestamp for the account bearing a given user ID.
+
+        Args:
+            user (str): The ID of the user.
+        Returns:
+            defer.Deferred: None, if the account has no expiration timestamp,
+            otherwise int representation of the timestamp (as a number of
+            milliseconds since epoch).
+        """
+        res = yield self._simple_select_one_onecol(
+            table="account_validity",
+            keyvalues={"user_id": user.to_string()},
+            retcol="expiration_ts_ms",
+            allow_none=True,
+            desc="get_expiration_date_for_user",
+        )
+        defer.returnValue(res)
+
     @defer.inlineCallbacks
     def is_server_admin(self, user):
         res = yield self._simple_select_one_onecol(
@@ -425,6 +445,8 @@ class RegistrationStore(
             columns=["creation_ts"],
         )
 
+        self._account_validity = hs.config.account_validity
+
         # we no longer use refresh tokens, but it's possible that some people
         # might have a background update queued to build this index. Just
         # clear the background update.
@@ -561,6 +583,18 @@ class RegistrationStore(
                         "user_type": user_type,
                     },
                 )
+
+            if self._account_validity.enabled:
+                now_ms = self.clock.time_msec()
+                expiration_ts = now_ms + self._account_validity.period
+                self._simple_insert_txn(
+                    txn,
+                    "account_validity",
+                    values={
+                        "user_id": user_id,
+                        "expiration_ts_ms": expiration_ts,
+                    }
+                )
         except self.database_engine.module.IntegrityError:
             raise StoreError(400, "User ID already taken.", errcode=Codes.USER_IN_USE)
 
diff --git a/synapse/storage/schema/delta/54/account_validity.sql b/synapse/storage/schema/delta/54/account_validity.sql
new file mode 100644
index 000000000..57249262d
--- /dev/null
+++ b/synapse/storage/schema/delta/54/account_validity.sql
@@ -0,0 +1,20 @@
+/* Copyright 2019 New Vector Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+-- Track what users are in public rooms.
+CREATE TABLE IF NOT EXISTS account_validity (
+    user_id TEXT PRIMARY KEY,
+    expiration_ts_ms BIGINT NOT NULL
+);
diff --git a/synapse/util/manhole.py b/synapse/util/manhole.py
index 9cb7e9c9a..628a2962d 100644
--- a/synapse/util/manhole.py
+++ b/synapse/util/manhole.py
@@ -1,4 +1,5 @@
 # Copyright 2016 OpenMarket Ltd
+# Copyright 2019 New Vector Ltd.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -11,10 +12,12 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import sys
+import traceback
 
 from twisted.conch import manhole_ssh
 from twisted.conch.insults import insults
-from twisted.conch.manhole import ColoredManhole
+from twisted.conch.manhole import ColoredManhole, ManholeInterpreter
 from twisted.conch.ssh.keys import Key
 from twisted.cred import checkers, portal
 
@@ -79,7 +82,7 @@ def manhole(username, password, globals):
 
     rlm = manhole_ssh.TerminalRealm()
     rlm.chainedProtocolFactory = lambda: insults.ServerProtocol(
-        ColoredManhole,
+        SynapseManhole,
         dict(globals, __name__="__console__")
     )
 
@@ -88,3 +91,55 @@ def manhole(username, password, globals):
     factory.privateKeys[b'ssh-rsa'] = Key.fromString(PRIVATE_KEY)
 
     return factory
+
+
+class SynapseManhole(ColoredManhole):
+    """Overrides connectionMade to create our own ManholeInterpreter"""
+    def connectionMade(self):
+        super(SynapseManhole, self).connectionMade()
+
+        # replace the manhole interpreter with our own impl
+        self.interpreter = SynapseManholeInterpreter(self, self.namespace)
+
+        # this would also be a good place to add more keyHandlers.
+
+
+class SynapseManholeInterpreter(ManholeInterpreter):
+    def showsyntaxerror(self, filename=None):
+        """Display the syntax error that just occurred.
+
+        Overrides the base implementation, ignoring sys.excepthook. We always want
+        any syntax errors to be sent to the terminal, rather than sentry.
+        """
+        type, value, tb = sys.exc_info()
+        sys.last_type = type
+        sys.last_value = value
+        sys.last_traceback = tb
+        if filename and type is SyntaxError:
+            # Work hard to stuff the correct filename in the exception
+            try:
+                msg, (dummy_filename, lineno, offset, line) = value.args
+            except ValueError:
+                # Not the format we expect; leave it alone
+                pass
+            else:
+                # Stuff in the right filename
+                value = SyntaxError(msg, (filename, lineno, offset, line))
+                sys.last_value = value
+        lines = traceback.format_exception_only(type, value)
+        self.write(''.join(lines))
+
+    def showtraceback(self):
+        """Display the exception that just occurred.
+
+        Overrides the base implementation, ignoring sys.excepthook. We always want
+        any syntax errors to be sent to the terminal, rather than sentry.
+        """
+        sys.last_type, sys.last_value, last_tb = ei = sys.exc_info()
+        sys.last_traceback = last_tb
+        try:
+            # We remove the first stack item because it is our own code.
+            lines = traceback.format_exception(ei[0], ei[1], last_tb.tb_next)
+            self.write(''.join(lines))
+        finally:
+            last_tb = ei = None


Test patch:
diff --git a/tests/rest/client/v2_alpha/test_register.py b/tests/rest/client/v2_alpha/test_register.py
index a45e6e5e1..d3611ed21 100644
--- a/tests/rest/client/v2_alpha/test_register.py
+++ b/tests/rest/client/v2_alpha/test_register.py
@@ -1,15 +1,18 @@
+import datetime
 import json
 
 from synapse.api.constants import LoginType
+from synapse.api.errors import Codes
 from synapse.appservice import ApplicationService
-from synapse.rest.client.v2_alpha.register import register_servlets
+from synapse.rest.client.v1 import admin, login
+from synapse.rest.client.v2_alpha import register, sync
 
 from tests import unittest
 
 
 class RegisterRestServletTestCase(unittest.HomeserverTestCase):
 
-    servlets = [register_servlets]
+    servlets = [register.register_servlets]
 
     def make_homeserver(self, reactor, clock):
 
@@ -181,3 +184,47 @@ class RegisterRestServletTestCase(unittest.HomeserverTestCase):
         self.render(request)
 
         self.assertEquals(channel.result["code"], b"200", channel.result)
+
+
+class AccountValidityTestCase(unittest.HomeserverTestCase):
+
+    servlets = [
+        register.register_servlets,
+        admin.register_servlets,
+        login.register_servlets,
+        sync.register_servlets,
+    ]
+
+    def make_homeserver(self, reactor, clock):
+        config = self.default_config()
+        config.enable_registration = True
+        config.account_validity.enabled = True
+        config.account_validity.period = 604800000  # Time in ms for 1 week
+        self.hs = self.setup_test_homeserver(config=config)
+
+        return self.hs
+
+    def test_validity_period(self):
+        self.register_user("kermit", "monkey")
+        tok = self.login("kermit", "monkey")
+
+        # The specific endpoint doesn't matter, all we need is an authenticated
+        # endpoint.
+        request, channel = self.make_request(
+            b"GET", "/sync", access_token=tok,
+        )
+        self.render(request)
+
+        self.assertEquals(channel.result["code"], b"200", channel.result)
+
+        self.reactor.advance(datetime.timedelta(weeks=1).total_seconds())
+
+        request, channel = self.make_request(
+            b"GET", "/sync", access_token=tok,
+        )
+        self.render(request)
+
+        self.assertEquals(channel.result["code"], b"403", channel.result)
+        self.assertEquals(
+            channel.json_body["errcode"], Codes.EXPIRED_ACCOUNT, channel.result,
+        )
diff --git a/tests/test_state.py b/tests/test_state.py
index 03e4810c2..5bcc6aaa1 100644
--- a/tests/test_state.py
+++ b/tests/test_state.py
@@ -25,7 +25,7 @@ from synapse.state import StateHandler, StateResolutionHandler
 
 from tests import unittest
 
-from .utils import MockClock
+from .utils import MockClock, default_config
 
 _next_event_id = 1000
 
@@ -160,6 +160,7 @@ class StateTestCase(unittest.TestCase):
         self.store = StateGroupStore()
         hs = Mock(
             spec_set=[
+                "config",
                 "get_datastore",
                 "get_auth",
                 "get_state_handler",
@@ -167,6 +168,7 @@ class StateTestCase(unittest.TestCase):
                 "get_state_resolution_handler",
             ]
         )
+        hs.config = default_config("tesths")
         hs.get_datastore.return_value = self.store
         hs.get_state_handler.return_value = None
         hs.get_clock.return_value = MockClock()