Repository: matrix-org/synapse

Issue:
/publicRooms fails on python3 with 500 - KeyError
Logs:
```
2018-10-15 18:40:40,840 - synapse.util.caches.response_cache - 151 - INFO - POST-117277- [room_list]: no cached result for [(20, 'hKFzzgCr+fuhcM0BXqFuE6Fkww', ThirdPartyInstanceID(appservice_id=None, network_id=None))], calculating new one
2018-10-15 18:40:40,841 - synapse.http.server - 112 - ERROR - POST-117277- Failed handle request via <function JsonResource._async_render at 0x7f0027bf6a60>: <XForwardedForRequest at 0x7effd5b46080 method='POST' uri='/_matrix/client/r0/publicRooms?' clientproto='HTTP/1.0' site=8008>: Traceback (most recent call last):
  File "/srv/matrix/synapse/lib/python3.6/site-packages/synapse/util/caches/response_cache.py", line 152, in wrap
    d = run_in_background(callback, *args, **kwargs)
  File "/srv/matrix/synapse/lib/python3.6/site-packages/synapse/util/logcontext.py", line 497, in run_in_background
    res = f(*args, **kwargs)
  File "/srv/matrix/synapse/lib/python3.6/site-packages/twisted/internet/defer.py", line 1613, in unwindGenerator
    return _cancellableInlineCallbacks(gen)
  File "/srv/matrix/synapse/lib/python3.6/site-packages/twisted/internet/defer.py", line 1529, in _cancellableInlineCallbacks
    _inlineCallbacks(None, g, status)
--- <exception caught here> ---
  File "/srv/matrix/synapse/lib/python3.6/site-packages/synapse/http/server.py", line 81, in wrapped_request_handler
    yield h(self, request)
  File "/srv/matrix/synapse/lib/python3.6/site-packages/synapse/http/server.py", line 316, in _async_render
    callback_return = yield callback(request, **kwargs)
  File "/srv/matrix/synapse/lib/python3.6/site-packages/synapse/rest/client/v1/room.py", line 371, in on_POST
    network_tuple=network_tuple,
  File "/srv/matrix/synapse/lib/python3.6/site-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks
    result = g.send(result)
  File "/srv/matrix/synapse/lib/python3.6/site-packages/synapse/handlers/room_list.py", line 92, in _get_public_room_list
    since_token = RoomListNextBatch.from_token(since_token)
  File "/srv/matrix/synapse/lib/python3.6/site-packages/synapse/handlers/room_list.py", line 449, in from_token
    for key, val in msgpack.loads(decode_base64(token)).items()
  File "/srv/matrix/synapse/lib/python3.6/site-packages/synapse/handlers/room_list.py", line 449, in <dictcomp>
    for key, val in msgpack.loads(decode_base64(token)).items()
builtins.KeyError: b's'
2018-10-15 18:40:40,842 - synapse.access.http.8008 - 302 - INFO - POST-117277- - - 8008 - {@Half-Shot:half-shot.uk} Processed request: 0.002sec/0.000sec (0.000sec, 0.000sec) (0.000sec/0.000sec/0) 55B 500 "POST /_matrix/client/r0/publicRooms? HTTP/1.0" "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Riot/0.16.5 Chrome/61.0.3163.100 Electron/2.0.8 Safari/537.36" [0 dbevts]
```


Discussion:

Resolving patch:
diff --git a/synapse/handlers/room_list.py b/synapse/handlers/room_list.py
--- a/synapse/handlers/room_list.py
+++ b/synapse/handlers/room_list.py
@@ -16,7 +16,7 @@
 import logging
 from collections import namedtuple
 
-from six import iteritems
+from six import PY3, iteritems
 from six.moves import range
 
 import msgpack
@@ -444,9 +444,16 @@ class RoomListNextBatch(namedtuple("RoomListNextBatch", (
 
     @classmethod
     def from_token(cls, token):
+        if PY3:
+            # The argument raw=False is only available on new versions of
+            # msgpack, and only really needed on Python 3. Gate it behind
+            # a PY3 check to avoid causing issues on Debian-packaged versions.
+            decoded = msgpack.loads(decode_base64(token), raw=False)
+        else:
+            decoded = msgpack.loads(decode_base64(token))
         return RoomListNextBatch(**{
             cls.REVERSE_KEY_DICT[key]: val
-            for key, val in msgpack.loads(decode_base64(token)).items()
+            for key, val in decoded.items()
         })
 
     def to_token(self):
diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py
--- a/synapse/python_dependencies.py
+++ b/synapse/python_dependencies.py
@@ -55,7 +55,7 @@
     "sortedcontainers>=1.4.4": ["sortedcontainers"],
     "pysaml2>=3.0.0": ["saml2"],
     "pymacaroons-pynacl>=0.9.3": ["pymacaroons"],
-    "msgpack-python>=0.3.0": ["msgpack"],
+    "msgpack-python>=0.4.2": ["msgpack"],
     "phonenumbers>=8.2.0": ["phonenumbers"],
     "six>=1.10": ["six"],
 


Test patch:
diff --git a/tests/handlers/test_roomlist.py b/tests/handlers/test_roomlist.py
new file mode 100644
--- /dev/null
+++ b/tests/handlers/test_roomlist.py
@@ -0,0 +1,39 @@
+# -*- coding: utf-8 -*-
+# Copyright 2018 New Vector Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from synapse.handlers.room_list import RoomListNextBatch
+
+import tests.unittest
+import tests.utils
+
+
+class RoomListTestCase(tests.unittest.TestCase):
+    """ Tests RoomList's RoomListNextBatch. """
+
+    def setUp(self):
+        pass
+
+    def test_check_read_batch_tokens(self):
+        batch_token = RoomListNextBatch(
+            stream_ordering="abcdef",
+            public_room_stream_id="123",
+            current_limit=20,
+            direction_is_forward=True,
+        ).to_token()
+        next_batch = RoomListNextBatch.from_token(batch_token)
+        self.assertEquals(next_batch.stream_ordering, "abcdef")
+        self.assertEquals(next_batch.public_room_stream_id, "123")
+        self.assertEquals(next_batch.current_limit, 20)
+        self.assertEquals(next_batch.direction_is_forward, True)