Repository: matrix-org/synapse

Issue:
"TypeError: 'in <string>' requires string as left operand, not bytes" when http[s] proxy is defined
Hi
Since synapse 1.6.0 supports outbound http proxy, there's an issue starting it if a proxy is defined with a port.

My env has https_proxy=10.99.2.1:3128 (set from systemd). And after an upgrade to Synapse 1.6.0, it fails to start with this :

```
Nov 26 15:50:27 mtx matrix-synapse[25593]: File "/opt/matrix/synapse/lib/python3.6/site-packages/synapse/app/_base.py", line 271, in start
Nov 26 15:50:27 mtx matrix-synapse[25593]: hs.start_listening(listeners)
Nov 26 15:50:27 mtx matrix-synapse[25593]: File "/opt/matrix/synapse/lib/python3.6/site-packages/synapse/app/homeserver.py", line 267, in start_listening
Nov 26 15:50:27 mtx matrix-synapse[25593]: self._listening_services.extend(self._listener_http(config, listener))
Nov 26 15:50:27 mtx matrix-synapse[25593]: File "/opt/matrix/synapse/lib/python3.6/site-packages/synapse/app/homeserver.py", line 105, in _listener_http
Nov 26 15:50:27 mtx matrix-synapse[25593]: self._configure_named_resource(name, res.get("compress", False))
Nov 26 15:50:27 mtx matrix-synapse[25593]: File "/opt/matrix/synapse/lib/python3.6/site-packages/synapse/app/homeserver.py", line 172, in _configure_named_resource
Nov 26 15:50:27 mtx matrix-synapse[25593]: client_resource = ClientRestResource(self)
Nov 26 15:50:27 mtx matrix-synapse[25593]: File "/opt/matrix/synapse/lib/python3.6/site-packages/synapse/rest/__init__.py", line 72, in __init__
Nov 26 15:50:27 mtx matrix-synapse[25593]: self.register_servlets(self, hs)
Nov 26 15:50:27 mtx matrix-synapse[25593]: File "/opt/matrix/synapse/lib/python3.6/site-packages/synapse/rest/__init__.py", line 83, in register_servlets
Nov 26 15:50:27 mtx matrix-synapse[25593]: events.register_servlets(hs, client_resource)
Nov 26 15:50:27 mtx matrix-synapse[25593]: File "/opt/matrix/synapse/lib/python3.6/site-packages/synapse/rest/client/v1/events.py", line 100, in register_servlets
Nov 26 15:50:27 mtx matrix-synapse[25593]: EventStreamRestServlet(hs).register(http_server)
Nov 26 15:50:27 mtx matrix-synapse[25593]: File "/opt/matrix/synapse/lib/python3.6/site-packages/synapse/rest/client/v1/events.py", line 36, in __init__
Nov 26 15:50:27 mtx matrix-synapse[25593]: self.event_stream_handler = hs.get_event_stream_handler()
Nov 26 15:50:27 mtx matrix-synapse[25593]: File "/opt/matrix/synapse/lib/python3.6/site-packages/synapse/server.py", line 586, in _get
Nov 26 15:50:27 mtx matrix-synapse[25593]: dep = builder()
Nov 26 15:50:27 mtx matrix-synapse[25593]: File "/opt/matrix/synapse/lib/python3.6/site-packages/synapse/server.py", line 386, in build_event_stream_handler
Nov 26 15:50:27 mtx matrix-synapse[25593]: return EventStreamHandler(self)
Nov 26 15:50:27 mtx matrix-synapse[25593]: File "/opt/matrix/synapse/lib/python3.6/site-packages/synapse/handlers/events.py", line 50, in __init__
Nov 26 15:50:27 mtx matrix-synapse[25593]: self._server_notices_sender = hs.get_server_notices_sender()
Nov 26 15:50:27 mtx matrix-synapse[25593]: File "/opt/matrix/synapse/lib/python3.6/site-packages/synapse/server.py", line 586, in _get
Nov 26 15:50:27 mtx matrix-synapse[25593]: dep = builder()
Nov 26 15:50:27 mtx matrix-synapse[25593]: File "/opt/matrix/synapse/lib/python3.6/site-packages/synapse/server.py", line 529, in build_server_notices_sender
Nov 26 15:50:27 mtx matrix-synapse[25593]: return ServerNoticesSender(self)
Nov 26 15:50:27 mtx matrix-synapse[25593]: File "/opt/matrix/synapse/lib/python3.6/site-packages/synapse/server_notices/server_notices_sender.py", line 35, in __init__
Nov 26 15:50:27 mtx matrix-synapse[25593]: ConsentServerNotices(hs),
Nov 26 15:50:27 mtx matrix-synapse[25593]: File "/opt/matrix/synapse/lib/python3.6/site-packages/synapse/server_notices/consent_server_notices.py", line 40, in __init__
Nov 26 15:50:27 mtx matrix-synapse[25593]: self._server_notices_manager = hs.get_server_notices_manager()
Nov 26 15:50:27 mtx matrix-synapse[25593]: File "/opt/matrix/synapse/lib/python3.6/site-packages/synapse/server.py", line 586, in _get
Nov 26 15:50:27 mtx matrix-synapse[25593]: dep = builder()
Nov 26 15:50:27 mtx matrix-synapse[25593]: File "/opt/matrix/synapse/lib/python3.6/site-packages/synapse/server.py", line 524, in build_server_notices_manager
Nov 26 15:50:27 mtx matrix-synapse[25593]: return ServerNoticesManager(self)
Nov 26 15:50:27 mtx matrix-synapse[25593]: File "/opt/matrix/synapse/lib/python3.6/site-packages/synapse/server_notices/server_notices_manager.py", line 38, in __init__
Nov 26 15:50:27 mtx matrix-synapse[25593]: self._room_creation_handler = hs.get_room_creation_handler()
Nov 26 15:50:27 mtx matrix-synapse[25593]: File "/opt/matrix/synapse/lib/python3.6/site-packages/synapse/server.py", line 586, in _get
Nov 26 15:50:27 mtx matrix-synapse[25593]: dep = builder()
Nov 26 15:50:27 mtx matrix-synapse[25593]: File "/opt/matrix/synapse/lib/python3.6/site-packages/synapse/server.py", line 326, in build_room_creation_handler
Nov 26 15:50:27 mtx matrix-synapse[25593]: return RoomCreationHandler(self)
Nov 26 15:50:27 mtx matrix-synapse[25593]: File "/opt/matrix/synapse/lib/python3.6/site-packages/synapse/handlers/room.py", line 76, in __init__
Nov 26 15:50:27 mtx matrix-synapse[25593]: self.room_member_handler = hs.get_room_member_handler()
Nov 26 15:50:27 mtx matrix-synapse[25593]: File "/opt/matrix/synapse/lib/python3.6/site-packages/synapse/server.py", line 586, in _get
Nov 26 15:50:27 mtx matrix-synapse[25593]: dep = builder()
Nov 26 15:50:27 mtx matrix-synapse[25593]: File "/opt/matrix/synapse/lib/python3.6/site-packages/synapse/server.py", line 513, in build_room_member_handler
Nov 26 15:50:27 mtx matrix-synapse[25593]: return RoomMemberMasterHandler(self)
Nov 26 15:50:27 mtx matrix-synapse[25593]: File "/opt/matrix/synapse/lib/python3.6/site-packages/synapse/handlers/room_member.py", line 871, in __init__
Nov 26 15:50:27 mtx matrix-synapse[25593]: super(RoomMemberMasterHandler, self).__init__(hs)
Nov 26 15:50:27 mtx matrix-synapse[25593]: File "/opt/matrix/synapse/lib/python3.6/site-packages/synapse/handlers/room_member.py", line 60, in __init__
Nov 26 15:50:27 mtx matrix-synapse[25593]: self.registration_handler = hs.get_registration_handler()
Nov 26 15:50:27 mtx matrix-synapse[25593]: File "/opt/matrix/synapse/lib/python3.6/site-packages/synapse/server.py", line 586, in _get
Nov 26 15:50:27 mtx matrix-synapse[25593]: dep = builder()
Nov 26 15:50:27 mtx matrix-synapse[25593]: File "/opt/matrix/synapse/lib/python3.6/site-packages/synapse/server.py", line 541, in build_registration_handler
Nov 26 15:50:27 mtx matrix-synapse[25593]: return RegistrationHandler(self)
Nov 26 15:50:27 mtx matrix-synapse[25593]: File "/opt/matrix/synapse/lib/python3.6/site-packages/synapse/handlers/register.py", line 55, in __init__
Nov 26 15:50:27 mtx matrix-synapse[25593]: self._auth_handler = hs.get_auth_handler()
Nov 26 15:50:27 mtx matrix-synapse[25593]: File "/opt/matrix/synapse/lib/python3.6/site-packages/synapse/server.py", line 586, in _get
Nov 26 15:50:27 mtx matrix-synapse[25593]: dep = builder()
Nov 26 15:50:27 mtx matrix-synapse[25593]: File "/opt/matrix/synapse/lib/python3.6/site-packages/synapse/server.py", line 350, in build_auth_handler
Nov 26 15:50:27 mtx matrix-synapse[25593]: return AuthHandler(self)
Nov 26 15:50:27 mtx matrix-synapse[25593]: File "/opt/matrix/synapse/lib/python3.6/site-packages/synapse/handlers/auth.py", line 63, in __init__
Nov 26 15:50:27 mtx matrix-synapse[25593]: inst = auth_checker_class(hs)
Nov 26 15:50:27 mtx matrix-synapse[25593]: File "/opt/matrix/synapse/lib/python3.6/site-packages/synapse/handlers/ui_auth/checkers.py", line 84, in __init__
Nov 26 15:50:27 mtx matrix-synapse[25593]: self._http_client = hs.get_proxied_http_client()
Nov 26 15:50:27 mtx matrix-synapse[25593]: File "/opt/matrix/synapse/lib/python3.6/site-packages/synapse/server.py", line 586, in _get
Nov 26 15:50:27 mtx matrix-synapse[25593]: dep = builder()
Nov 26 15:50:27 mtx matrix-synapse[25593]: File "/opt/matrix/synapse/lib/python3.6/site-packages/synapse/server.py", line 322, in build_proxied_http_client
Nov 26 15:50:27 mtx matrix-synapse[25593]: https_proxy=os.getenv("HTTPS_PROXY"),
Nov 26 15:50:27 mtx matrix-synapse[25593]: File "/opt/matrix/synapse/lib/python3.6/site-packages/synapse/http/client.py", line 256, in __init__
Nov 26 15:50:27 mtx matrix-synapse[25593]: https_proxy=https_proxy,
Nov 26 15:50:27 mtx matrix-synapse[25593]: File "/opt/matrix/synapse/lib/python3.6/site-packages/synapse/http/proxyagent.py", line 75, in __init__
Nov 26 15:50:27 mtx matrix-synapse[25593]: http_proxy, reactor, **self._endpoint_kwargs
Nov 26 15:50:27 mtx matrix-synapse[25593]: File "/opt/matrix/synapse/lib/python3.6/site-packages/synapse/http/proxyagent.py", line 179, in _http_proxy_endpoint
Nov 26 15:50:27 mtx matrix-synapse[25593]: host, port = parse_host_port(proxy, default_port=1080)
Nov 26 15:50:27 mtx matrix-synapse[25593]: File "/opt/matrix/synapse/lib/python3.6/site-packages/synapse/http/proxyagent.py", line 185, in parse_host_port
Nov 26 15:50:27 mtx matrix-synapse[25593]: if b":" in hostport:
Nov 26 15:50:27 mtx matrix-synapse[25593]: TypeError: 'in <string>' requires string as left operand, not bytes
Nov 26 15:50:27 mtx matrix-synapse[25593]: During handling of the above exception, another exception occurred:
Nov 26 15:50:27 mtx matrix-synapse[25593]: Traceback (most recent call last):
Nov 26 15:50:27 mtx matrix-synapse[25593]: File "/opt/matrix/synapse/lib/python3.6/site-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks
Nov 26 15:50:27 mtx matrix-synapse[25593]: result = g.send(result)
Nov 26 15:50:27 mtx matrix-synapse[25593]: File "/opt/matrix/synapse/lib/python3.6/site-packages/synapse/app/homeserver.py", line 436, in start
Nov 26 15:50:27 mtx matrix-synapse[25593]: _base.start(hs, config.listeners)
Nov 26 15:50:27 mtx matrix-synapse[25593]: File "/opt/matrix/synapse/lib/python3.6/site-packages/synapse/app/_base.py", line 281, in start
Nov 26 15:50:27 mtx matrix-synapse[25593]: sys.exit(1)
Nov 26 15:50:27 mtx matrix-synapse[25593]: SystemExit: 1
```

The problem is in synapse/http/proxyagent.py lines 185 and 186
```
    if b":" in hostport:
        host, port = hostport.rsplit(b":", 1)
```

Removing the b suffix, to work with strings instead of bytes fixes it for me.
I'm running on CentOS 7 with python 3.6.3 (in a virtual env)


Discussion:

Resolving patch:
diff --git a/synapse/rest/media/v1/preview_url_resource.py b/synapse/rest/media/v1/preview_url_resource.py
--- a/synapse/rest/media/v1/preview_url_resource.py
+++ b/synapse/rest/media/v1/preview_url_resource.py
@@ -77,8 +77,8 @@ def __init__(self, hs, media_repo, media_storage):
             treq_args={"browser_like_redirects": True},
             ip_whitelist=hs.config.url_preview_ip_range_whitelist,
             ip_blacklist=hs.config.url_preview_ip_range_blacklist,
-            http_proxy=os.getenv("http_proxy"),
-            https_proxy=os.getenv("HTTPS_PROXY"),
+            http_proxy=os.getenvb(b"http_proxy"),
+            https_proxy=os.getenvb(b"HTTPS_PROXY"),
         )
         self.media_repo = media_repo
         self.primary_base_path = media_repo.primary_base_path
diff --git a/synapse/server.py b/synapse/server.py
--- a/synapse/server.py
+++ b/synapse/server.py
@@ -318,8 +318,8 @@ def build_simple_http_client(self):
     def build_proxied_http_client(self):
         return SimpleHttpClient(
             self,
-            http_proxy=os.getenv("http_proxy"),
-            https_proxy=os.getenv("HTTPS_PROXY"),
+            http_proxy=os.getenvb(b"http_proxy"),
+            https_proxy=os.getenvb(b"HTTPS_PROXY"),
         )
 
     def build_room_creation_handler(self):


Test patch: