CVE-2021-29431: Improper Input Validation (CWE-20)
value: Sydent is a reference Matrix identity server. Sydent can be induced to send HTTP GET requests to internal systems, due to lack of parameter validation or IP address blacklisting. It is not possible to exfiltrate data or control request headers, but it might be possible to use the attack to perform an internal port enumeration. This issue has been addressed in in 9e57334, 8936925, 3d531ed, 0f00412. A potential workaround would be to use a firewall to ensure that Sydent cannot reach internal HTTP resources.
CWE description: The product receives input or data, but it does
        not validate or incorrectly validates that the input has the
        properties that are required to process the data safely and
        correctly.
CVSS v3 base score: 6.5
Published: 2021-04-15T21:15Z
Repository: https://github.com/matrix-org/sydent
Language: Python
Files changed: matrix_is_test/launcher.py, sydent/http/blacklisting_reactor.py, sydent/http/httpclient.py, sydent/sydent.py, sydent/util/ip_range.py, tests/test_blacklisting.py, tests/utils.py
Commit date: 2021-04-06 12:49:29 +0000
Commit message: Add BlacklistingReactor

This is a port from the Synapse codebase.

Diff: diff --git a/matrix_is_test/launcher.py b/matrix_is_test/launcher.py
index 86a6a78..d506263 100644
--- a/matrix_is_test/launcher.py
+++ b/matrix_is_test/launcher.py
@@ -34,10 +34,13 @@ db.file = :memory:
 server.name = test.local
 terms.path = {terms_path}
 templates.path = {testsubject_path}/res
 brand.default = is-test
 
+
+ip.whitelist = 127.0.0.1
+
 [email]
 email.tlsmode = 0
 email.invite.subject = %(sender_display_name)s has invited you to chat
 email.smtphost = localhost
 email.from = Sydent Validation <noreply@localhost>
diff --git a/sydent/http/blacklisting_reactor.py b/sydent/http/blacklisting_reactor.py
new file mode 100644
index 0000000..694b947
--- /dev/null
+++ b/sydent/http/blacklisting_reactor.py
@@ -0,0 +1,155 @@
+# -*- coding: utf-8 -*-
+# Copyright 2021 The Matrix.org Foundation C.I.C.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import logging
+from typing import (
+    Any,
+    List,
+    Optional,
+)
+
+from zope.interface import implementer, provider
+from netaddr import IPAddress, IPSet
+
+from twisted.internet.address import IPv4Address, IPv6Address
+from twisted.internet.interfaces import (
+    IAddress,
+    IHostResolution,
+    IReactorPluggableNameResolver,
+    IResolutionReceiver,
+)
+
+
+logger = logging.getLogger(__name__)
+
+
+def check_against_blacklist(
+    ip_address: IPAddress, ip_whitelist: Optional[IPSet], ip_blacklist: IPSet
+) -> bool:
+    """
+    Compares an IP address to allowed and disallowed IP sets.
+
+    Args:
+        ip_address: The IP address to check
+        ip_whitelist: Allowed IP addresses.
+        ip_blacklist: Disallowed IP addresses.
+
+    Returns:
+        True if the IP address is in the blacklist and not in the whitelist.
+    """
+    if ip_address in ip_blacklist:
+        if ip_whitelist is None or ip_address not in ip_whitelist:
+            return True
+    return False
+
+
+class _IPBlacklistingResolver:
+    """
+    A proxy for reactor.nameResolver which only produces non-blacklisted IP
+    addresses, preventing DNS rebinding attacks on URL preview.
+    """
+
+    def __init__(
+        self,
+        reactor: IReactorPluggableNameResolver,
+        ip_whitelist: Optional[IPSet],
+        ip_blacklist: IPSet,
+    ):
+        """
+        Args:
+            reactor: The twisted reactor.
+            ip_whitelist: IP addresses to allow.
+            ip_blacklist: IP addresses to disallow.
+        """
+        self._reactor = reactor
+        self._ip_whitelist = ip_whitelist
+        self._ip_blacklist = ip_blacklist
+
+    def resolveHostName(
+        self, recv: IResolutionReceiver, hostname: str, portNumber: int = 0
+    ) -> IResolutionReceiver:
+        addresses = []  # type: List[IAddress]
+
+        def _callback() -> None:
+            has_bad_ip = False
+            for address in addresses:
+                # We only expect IPv4 and IPv6 addresses since only A/AAAA lookups
+                # should go through this path.
+                if not isinstance(address, (IPv4Address, IPv6Address)):
+                    continue
+
+                ip_address = IPAddress(address.host)
+
+                if check_against_blacklist(
+                    ip_address, self._ip_whitelist, self._ip_blacklist
+                ):
+                    logger.info(
+                        "Dropped %s from DNS resolution to %s due to blacklist"
+                        % (ip_address, hostname)
+                    )
+                    has_bad_ip = True
+
+            # if we have a blacklisted IP, we'd like to raise an error to block the
+            # request, but all we can really do from here is claim that there were no
+            # valid results.
+            if not has_bad_ip:
+                for address in addresses:
+                    recv.addressResolved(address)
+            recv.resolutionComplete()
+
+        @provider(IResolutionReceiver)
+        class EndpointReceiver:
+            @staticmethod
+            def resolutionBegan(resolutionInProgress: IHostResolution) -> None:
+                recv.resolutionBegan(resolutionInProgress)
+
+            @staticmethod
+            def addressResolved(address: IAddress) -> None:
+                addresses.append(address)
+
+            @staticmethod
+            def resolutionComplete() -> None:
+                _callback()
+
+        self._reactor.nameResolver.resolveHostName(
+            EndpointReceiver, hostname, portNumber=portNumber
+        )
+
+        return recv
+
+
+@implementer(IReactorPluggableNameResolver)
+class BlacklistingReactorWrapper:
+    """
+    A Reactor wrapper which will prevent DNS resolution to blacklisted IP
+    addresses, to prevent DNS rebinding.
+    """
+
+    def __init__(
+        self,
+        reactor: IReactorPluggableNameResolver,
+        ip_whitelist: Optional[IPSet],
+        ip_blacklist: IPSet,
+    ):
+        self._reactor = reactor
+
+        # We need to use a DNS resolver which filters out blacklisted IP
+        # addresses, to prevent DNS rebinding.
+        self.nameResolver = _IPBlacklistingResolver(
+            self._reactor, ip_whitelist, ip_blacklist
+        )
+
+    def __getattr__(self, attr: str) -> Any:
+        # Passthrough to the real reactor except for the DNS resolver.
+        return getattr(self._reactor, attr)
diff --git a/sydent/http/httpclient.py b/sydent/http/httpclient.py
index 2da0c5f..3044779 100644
--- a/sydent/http/httpclient.py
+++ b/sydent/http/httpclient.py
@@ -20,12 +20,13 @@ import logging
 from io import BytesIO
 
 from twisted.internet import defer
 from twisted.web.client import FileBodyProducer, Agent, readBody
 from twisted.web.http_headers import Headers
-from sydent.http.matrixfederationagent import MatrixFederationAgent
 
+from sydent.http.blacklisting_reactor import BlacklistingReactorWrapper
+from sydent.http.matrixfederationagent import MatrixFederationAgent
 from sydent.http.federation_tls_options import ClientTLSOptionsFactory
 from sydent.http.httpcommon import BodyExceededMaxSize, read_body_with_max_size
 
 logger = logging.getLogger(__name__)
 
@@ -114,19 +115,27 @@ class SimpleHttpClient(HTTPClient):
         self.sydent = sydent
         # The default endpoint factory in Twisted 14.0.0 (which we require) uses the
         # BrowserLikePolicyForHTTPS context factory which will do regular cert validation
         # 'like a browser'
         self.agent = Agent(
-            self.sydent.reactor,
+            BlacklistingReactorWrapper(
+                reactor=self.sydent.reactor,
+                ip_whitelist=sydent.ip_whitelist,
+                ip_blacklist=sydent.ip_blacklist,
+            ),
             connectTimeout=15,
         )
 
 class FederationHttpClient(HTTPClient):
     """HTTP client for federation requests to homeservers. Uses a
     MatrixFederationAgent.
     """
     def __init__(self, sydent):
         self.sydent = sydent
         self.agent = MatrixFederationAgent(
-            self.sydent.reactor,
-            ClientTLSOptionsFactory(sydent.cfg),
+            BlacklistingReactorWrapper(
+                reactor=self.sydent.reactor,
+                ip_whitelist=sydent.ip_whitelist,
+                ip_blacklist=sydent.ip_blacklist,
+            ),
+            ClientTLSOptionsFactory(sydent.cfg) if sydent.use_tls_for_federation else None,
         )
diff --git a/sydent/sydent.py b/sydent/sydent.py
index e842715..f6c4eef 100644
--- a/sydent/sydent.py
+++ b/sydent/sydent.py
@@ -22,10 +22,11 @@ import gc
 from six.moves import configparser
 import copy
 import logging
 import logging.handlers
 import os
+from typing import Set
 
 import twisted.internet.reactor
 from twisted.internet import task
 from twisted.python import log
 
@@ -44,10 +45,11 @@ from sydent.validators.emailvalidator import EmailValidator
 from sydent.validators.msisdnvalidator import MsisdnValidator
 from sydent.hs_federation.verifier import Verifier
 
 from sydent.util.hash import sha256_and_url_safe_base64
 from sydent.util.tokenutils import generateAlphanumericTokenOfLength
+from sydent.util.ip_range import generate_ip_set, DEFAULT_IP_RANGE_BLACKLIST
 
 from sydent.sign.ed25519 import SydentEd25519
 
 from sydent.http.servlets.emailservlet import EmailRequestCodeServlet, EmailValidateCodeServlet
 from sydent.http.servlets.msisdnservlet import MsisdnRequestCodeServlet, MsisdnValidateCodeServlet
@@ -105,10 +107,30 @@ CONFIG_DEFAULTS = {
         # 'sentry_dsn': 'https://...'  # The DSN has configured in the sentry instance project.
 
         # Whether clients and homeservers can register an association using v1 endpoints.
         'enable_v1_associations': 'true',
         'delete_tokens_on_bind': 'true',
+
+        # Prevent outgoing requests from being sent to the following blacklisted
+        # IP address CIDR ranges. If this option is not specified or empty then
+        # it defaults to private IP address ranges.
+        #
+        # The blacklist applies to all outbound requests except replication
+        # requests.
+        #
+        # (0.0.0.0 and :: are always blacklisted, whether or not they are
+        # explicitly listed here, since they correspond to unroutable
+        # addresses.)
+        'ip.blacklist': '',
+
+        # List of IP address CIDR ranges that should be allowed for outbound
+        # requests. This is useful for specifying exceptions to wide-ranging
+        # blacklisted target IP ranges.
+        #
+        # This whitelist overrides `ip.blacklist` and defaults to an empty
+        # list.
+        'ip.whitelist': '',
     },
     'db': {
         'db.file': os.environ.get('SYDENT_DB_PATH', 'sydent.db'),
     },
     'http': {
@@ -181,13 +203,14 @@ CONFIG_DEFAULTS = {
     },
 }
 
 
 class Sydent:
-    def __init__(self, cfg, reactor=twisted.internet.reactor):
+    def __init__(self, cfg, reactor=twisted.internet.reactor, use_tls_for_federation=True):
         self.reactor = reactor
         self.config_file = get_config_file_path()
+        self.use_tls_for_federation = use_tls_for_federation
 
         self.cfg = cfg
 
         logger.info("Starting Sydent server")
 
@@ -239,10 +262,19 @@ class Sydent:
 
         self.delete_tokens_on_bind = parse_cfg_bool(
             self.cfg.get("general", "delete_tokens_on_bind")
         )
 
+        ip_blacklist = set_from_comma_sep_string(self.cfg.get("general", "ip.blacklist"))
+        if not ip_blacklist:
+            ip_blacklist = DEFAULT_IP_RANGE_BLACKLIST
+
+        ip_whitelist = set_from_comma_sep_string(self.cfg.get("general", "ip.whitelist"))
+
+        self.ip_blacklist = generate_ip_set(ip_blacklist)
+        self.ip_whitelist = generate_ip_set(ip_whitelist)
+
         self.default_web_client_location = self.cfg.get(
             "email", "email.default_web_client_location"
         )
         self.username_obfuscate_characters = int(self.cfg.get(
             "email", "email.third_party_invite_username_obfuscate_characters"
@@ -510,10 +542,16 @@ def get_config_file_path():
 
 def parse_cfg_bool(value):
     return value.lower() == "true"
 
 
+def set_from_comma_sep_string(rawstr: str) -> Set[str]:
+    if rawstr == '':
+        return set()
+    return {x.strip() for x in rawstr.split(',')}
+
+
 def run_gc():
     threshold = gc.get_threshold()
     counts = gc.get_count()
     for i in reversed(range(len(threshold))):
         if threshold[i] < counts[i]:
diff --git a/sydent/util/ip_range.py b/sydent/util/ip_range.py
new file mode 100644
index 0000000..9e81a63
--- /dev/null
+++ b/sydent/util/ip_range.py
@@ -0,0 +1,118 @@
+#  Copyright 2021 The Matrix.org Foundation C.I.C.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+import itertools
+from typing import Iterable, Optional
+
+from netaddr import AddrFormatError, IPNetwork, IPSet
+
+# IP ranges that are considered private / unroutable / don't make sense.
+DEFAULT_IP_RANGE_BLACKLIST = [
+    # Localhost
+    "127.0.0.0/8",
+    # Private networks.
+    "10.0.0.0/8",
+    "172.16.0.0/12",
+    "192.168.0.0/16",
+    # Carrier grade NAT.
+    "100.64.0.0/10",
+    # Address registry.
+    "192.0.0.0/24",
+    # Link-local networks.
+    "169.254.0.0/16",
+    # Formerly used for 6to4 relay.
+    "192.88.99.0/24",
+    # Testing networks.
+    "198.18.0.0/15",
+    "192.0.2.0/24",
+    "198.51.100.0/24",
+    "203.0.113.0/24",
+    # Multicast.
+    "224.0.0.0/4",
+    # Localhost
+    "::1/128",
+    # Link-local addresses.
+    "fe80::/10",
+    # Unique local addresses.
+    "fc00::/7",
+    # Testing networks.
+    "2001:db8::/32",
+    # Multicast.
+    "ff00::/8",
+    # Site-local addresses
+    "fec0::/10",
+]
+
+
+def generate_ip_set(
+    ip_addresses: Optional[Iterable[str]],
+    extra_addresses: Optional[Iterable[str]] = None,
+    config_path: Optional[Iterable[str]] = None,
+) -> IPSet:
+    """
+    Generate an IPSet from a list of IP addresses or CIDRs.
+
+    Additionally, for each IPv4 network in the list of IP addresses, also
+    includes the corresponding IPv6 networks.
+
+    This includes:
+
+    * IPv4-Compatible IPv6 Address (see RFC 4291, section 2.5.5.1)
+    * IPv4-Mapped IPv6 Address (see RFC 4291, section 2.5.5.2)
+    * 6to4 Address (see RFC 3056, section 2)
+
+    Args:
+        ip_addresses: An iterable of IP addresses or CIDRs.
+        extra_addresses: An iterable of IP addresses or CIDRs.
+        config_path: The path in the configuration for error messages.
+
+    Returns:
+        A new IP set.
+    """
+    result = IPSet()
+    for ip in itertools.chain(ip_addresses or (), extra_addresses or ()):
+        try:
+            network = IPNetwork(ip)
+        except AddrFormatError as e:
+            raise Exception(
+                "Invalid IP range provided: %s." % (ip,), config_path
+            ) from e
+        result.add(network)
+
+        # It is possible that these already exist in the set, but that's OK.
+        if ":" not in str(network):
+            result.add(IPNetwork(network).ipv6(ipv4_compatible=True))
+            result.add(IPNetwork(network).ipv6(ipv4_compatible=False))
+            result.add(_6to4(network))
+
+    return result
+
+
+def _6to4(network: IPNetwork) -> IPNetwork:
+    """Convert an IPv4 network into a 6to4 IPv6 network per RFC 3056."""
+
+    # 6to4 networks consist of:
+    # * 2002 as the first 16 bits
+    # * The first IPv4 address in the network hex-encoded as the next 32 bits
+    # * The new prefix length needs to include the bits from the 2002 prefix.
+    hex_network = hex(network.first)[2:]
+    hex_network = ("0" * (8 - len(hex_network))) + hex_network
+    return IPNetwork(
+        "2002:%s:%s::/%d"
+        % (
+            hex_network[:4],
+            hex_network[4:],
+            16 + network.prefixlen,
+        )
+    )
diff --git a/tests/test_blacklisting.py b/tests/test_blacklisting.py
new file mode 100644
index 0000000..e91c5d2
--- /dev/null
+++ b/tests/test_blacklisting.py
@@ -0,0 +1,243 @@
+#  Copyright 2021 The Matrix.org Foundation C.I.C.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+
+from mock import patch
+from netaddr import IPSet
+from twisted.internet import defer
+from twisted.internet.error import DNSLookupError
+from twisted.test.proto_helpers import StringTransport
+from twisted.trial.unittest import TestCase
+from twisted.web.client import Agent
+
+from sydent.http.blacklisting_reactor import BlacklistingReactorWrapper
+from sydent.http.srvresolver import Server
+from tests.utils import make_request, make_sydent
+
+
+class BlacklistingAgentTest(TestCase):
+    def setUp(self):
+        config = {
+            "general": {
+                "ip.blacklist": "5.0.0.0/8",
+                "ip.whitelist": "5.1.1.1",
+            },
+        }
+
+        self.sydent = make_sydent(test_config=config)
+
+        self.reactor = self.sydent.reactor
+
+        self.safe_domain, self.safe_ip = b"safe.test", b"1.2.3.4"
+        self.unsafe_domain, self.unsafe_ip = b"danger.test", b"5.6.7.8"
+        self.allowed_domain, self.allowed_ip = b"allowed.test", b"5.1.1.1"
+
+        # Configure the reactor's DNS resolver.
+        for (domain, ip) in (
+            (self.safe_domain, self.safe_ip),
+            (self.unsafe_domain, self.unsafe_ip),
+            (self.allowed_domain, self.allowed_ip),
+        ):
+            self.reactor.lookups[domain.decode()] = ip.decode()
+            self.reactor.lookups[ip.decode()] = ip.decode()
+
+        self.ip_whitelist = self.sydent.ip_whitelist
+        self.ip_blacklist = self.sydent.ip_blacklist
+
+    def test_reactor(self):
+        """Apply the blacklisting reactor and ensure it properly blocks
+        connections to particular domains and IPs.
+        """
+        agent = Agent(
+            BlacklistingReactorWrapper(
+                self.reactor,
+                ip_whitelist=self.ip_whitelist,
+                ip_blacklist=self.ip_blacklist,
+            ),
+        )
+
+        # The unsafe domains and IPs should be rejected.
+        for domain in (self.unsafe_domain, self.unsafe_ip):
+            self.failureResultOf(
+                agent.request(b"GET", b"http://" + domain), DNSLookupError
+            )
+
+        self.reactor.tcpClients = []
+
+        # The safe domains IPs should be accepted.
+        for domain in (
+            self.safe_domain,
+            self.allowed_domain,
+            self.safe_ip,
+            self.allowed_ip,
+        ):
+            agent.request(b"GET", b"http://" + domain)
+
+            # Grab the latest TCP connection.
+            (
+                host,
+                port,
+                client_factory,
+                _timeout,
+                _bindAddress,
+            ) = self.reactor.tcpClients.pop()
+
+    @patch("sydent.http.srvresolver.SrvResolver.resolve_service")
+    def test_federation_client_allowed_ip(self, resolver):
+        self.sydent.run()
+
+        request, channel = make_request(
+            self.sydent.reactor,
+            "POST",
+            "/_matrix/identity/v2/account/register",
+            {
+                "access_token": "foo",
+                "expires_in": 300,
+                "matrix_server_name": "example.com",
+                "token_type": "Bearer",
+            },
+        )
+
+        resolver.return_value = defer.succeed(
+            [
+                Server(
+                    host=self.allowed_domain,
+                    port=443,
+                    priority=1,
+                    weight=1,
+                    expires=100,
+                )
+            ]
+        )
+
+        request.render(self.sydent.servlets.registerServlet)
+
+        transport, protocol = self._get_http_request(
+            self.allowed_ip.decode("ascii"), 443
+        )
+
+        self.assertRegex(
+            transport.value(), b"^GET /_matrix/federation/v1/openid/userinfo"
+        )
+        self.assertRegex(transport.value(), b"Host: example.com")
+
+        # Send it the HTTP response
+        res_json = '{ "sub": "@test:example.com" }'.encode("ascii")
+        protocol.dataReceived(
+            b"HTTP/1.1 200 OK\r\n"
+            b"Server: Fake\r\n"
+            b"Content-Type: application/json\r\n"
+            b"Content-Length: %i\r\n"
+            b"\r\n"
+            b"%s" % (len(res_json), res_json)
+        )
+
+        self.assertEqual(channel.code, 200)
+
+    @patch("sydent.http.srvresolver.SrvResolver.resolve_service")
+    def test_federation_client_safe_ip(self, resolver):
+        self.sydent.run()
+
+        request, channel = make_request(
+            self.sydent.reactor,
+            "POST",
+            "/_matrix/identity/v2/account/register",
+            {
+                "access_token": "foo",
+                "expires_in": 300,
+                "matrix_server_name": "example.com",
+                "token_type": "Bearer",
+            },
+        )
+
+        resolver.return_value = defer.succeed(
+            [
+                Server(
+                    host=self.safe_domain,
+                    port=443,
+                    priority=1,
+                    weight=1,
+                    expires=100,
+                )
+            ]
+        )
+
+        request.render(self.sydent.servlets.registerServlet)
+
+        transport, protocol = self._get_http_request(self.safe_ip.decode("ascii"), 443)
+
+        self.assertRegex(
+            transport.value(), b"^GET /_matrix/federation/v1/openid/userinfo"
+        )
+        self.assertRegex(transport.value(), b"Host: example.com")
+
+        # Send it the HTTP response
+        res_json = '{ "sub": "@test:example.com" }'.encode("ascii")
+        protocol.dataReceived(
+            b"HTTP/1.1 200 OK\r\n"
+            b"Server: Fake\r\n"
+            b"Content-Type: application/json\r\n"
+            b"Content-Length: %i\r\n"
+            b"\r\n"
+            b"%s" % (len(res_json), res_json)
+        )
+
+        self.assertEqual(channel.code, 200)
+
+    @patch("sydent.http.srvresolver.SrvResolver.resolve_service")
+    def test_federation_client_unsafe_ip(self, resolver):
+        self.sydent.run()
+
+        request, channel = make_request(
+            self.sydent.reactor,
+            "POST",
+            "/_matrix/identity/v2/account/register",
+            {
+                "access_token": "foo",
+                "expires_in": 300,
+                "matrix_server_name": "example.com",
+                "token_type": "Bearer",
+            },
+        )
+
+        resolver.return_value = defer.succeed(
+            [
+                Server(
+                    host=self.unsafe_domain,
+                    port=443,
+                    priority=1,
+                    weight=1,
+                    expires=100,
+                )
+            ]
+        )
+
+        request.render(self.sydent.servlets.registerServlet)
+
+        self.assertNot(self.reactor.tcpClients)
+
+        self.assertEqual(channel.code, 500)
+
+    def _get_http_request(self, expected_host, expected_port):
+        clients = self.reactor.tcpClients
+        (host, port, factory, _timeout, _bindAddress) = clients[-1]
+        self.assertEqual(host, expected_host)
+        self.assertEqual(port, expected_port)
+
+        # complete the connection and wire it up to a fake transport
+        protocol = factory.buildProtocol(None)
+        transport = StringTransport()
+        protocol.makeConnection(transport)
+
+        return transport, protocol
diff --git a/tests/utils.py b/tests/utils.py
index 5104f49..ac01a78 100644
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -1,12 +1,22 @@
 import json
 from io import BytesIO
 import logging
 import os
-
+from typing import Dict
 import attr
 from six import text_type
+from zope.interface import implementer
+from twisted.internet._resolver import SimpleResolverComplexifier
+from twisted.internet.defer import fail, succeed
+from twisted.internet.error import DNSLookupError
+from twisted.internet.interfaces import (
+    IHostnameResolver,
+    IReactorPluggableNameResolver,
+    IResolverSimple,
+)
+
 from twisted.internet import address
 import twisted.logger
 from twisted.web.http_headers import Headers
 from twisted.web.server import Request, Site
 from twisted.web.http import unquote
@@ -51,17 +61,17 @@ def make_sydent(test_config={}):
             config
     """
     # Use an in-memory SQLite database. Note that the database isn't cleaned up between
     # tests, so by default the same database will be used for each test if changed to be
     # a file on disk.
-    if 'db' not in test_config:
-        test_config['db'] = {'db.file': ':memory:'}
+    if "db" not in test_config:
+        test_config["db"] = {"db.file": ":memory:"}
     else:
-        test_config['db'].setdefault('db.file', ':memory:')
+        test_config["db"].setdefault("db.file", ":memory:")
 
-    reactor = MemoryReactorClock()
-    return Sydent(reactor=reactor, cfg=parse_config_dict(test_config))
+    reactor = ResolvingMemoryReactorClock()
+    return Sydent(reactor=reactor, cfg=parse_config_dict(test_config), use_tls_for_federation=False)
 
 
 @attr.s
 class FakeChannel(object):
     """
@@ -147,10 +157,11 @@ class FakeChannel(object):
         return crypto.load_certificate(crypto.FILETYPE_PEM, FAKE_SERVER_CERT_PEM)
 
 
 class FakeSite:
     """A fake Twisted Web Site."""
+
     pass
 
 
 def make_request(
     reactor,
@@ -189,14 +200,11 @@ def make_request(
 
     if not isinstance(path, bytes):
         path = path.encode("ascii")
 
     # Decorate it to be the full path, if we're using shorthand
-    if (
-        shorthand
-        and not path.startswith(b"/_matrix")
-    ):
+    if shorthand and not path.startswith(b"/_matrix"):
         path = b"/_matrix/identity/v2/" + path
         path = path.replace(b"//", b"/")
 
     if not path.startswith(b"/"):
         path = b"/" + path
@@ -251,14 +259,11 @@ def setup_logging():
 
     (Logs will end up in _trial_temp.)
     """
     root_logger = logging.getLogger()
 
-    log_format = (
-        "%(asctime)s - %(name)s - %(lineno)d - %(levelname)s"
-        " - %(message)s"
-    )
+    log_format = "%(asctime)s - %(name)s - %(lineno)d - %(levelname)s" " - %(message)s"
 
     handler = ToTwistedHandler()
     formatter = logging.Formatter(log_format)
     handler.setFormatter(formatter)
     root_logger.addHandler(handler)
@@ -266,5 +271,28 @@ def setup_logging():
     log_level = os.environ.get("SYDENT_TEST_LOG_LEVEL", "ERROR")
     root_logger.setLevel(log_level)
 
 
 setup_logging()
+
+
+@implementer(IReactorPluggableNameResolver)
+class ResolvingMemoryReactorClock(MemoryReactorClock):
+    """
+    A MemoryReactorClock that supports name resolution.
+    """
+
+    def __init__(self):
+        lookups = self.lookups = {}  # type: Dict[str, str]
+
+        @implementer(IResolverSimple)
+        class FakeResolver:
+            def getHostByName(self, name, timeout=None):
+                if name not in lookups:
+                    return fail(DNSLookupError("OH NO: unknown %s" % (name,)))
+                return succeed(lookups[name])
+
+        self.nameResolver = SimpleResolverComplexifier(FakeResolver())
+        super().__init__()
+
+    def installNameResolver(self, resolver: IHostnameResolver) -> IHostnameResolver:
+        raise NotImplementedError()

Vulnerable code: from sydent.http.matrixfederationagent import MatrixFederationAgent
            self.sydent.reactor,
            self.sydent.reactor,
            ClientTLSOptionsFactory(sydent.cfg),
    def __init__(self, cfg, reactor=twisted.internet.reactor):

    if 'db' not in test_config:
        test_config['db'] = {'db.file': ':memory:'}
        test_config['db'].setdefault('db.file', ':memory:')
    reactor = MemoryReactorClock()
    return Sydent(reactor=reactor, cfg=parse_config_dict(test_config))
    if (
        shorthand
        and not path.startswith(b"/_matrix")
    ):
    log_format = (
        "%(asctime)s - %(name)s - %(lineno)d - %(levelname)s"
        " - %(message)s"
    )
Fixed code: 
ip.whitelist = 127.0.0.1

# -*- coding: utf-8 -*-
# Copyright 2021 The Matrix.org Foundation C.I.C.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import logging
from typing import (
    Any,
    List,
    Optional,
)

from zope.interface import implementer, provider
from netaddr import IPAddress, IPSet

from twisted.internet.address import IPv4Address, IPv6Address
from twisted.internet.interfaces import (
    IAddress,
    IHostResolution,
    IReactorPluggableNameResolver,
    IResolutionReceiver,
)


logger = logging.getLogger(__name__)


def check_against_blacklist(
    ip_address: IPAddress, ip_whitelist: Optional[IPSet], ip_blacklist: IPSet
) -> bool:
    """
    Compares an IP address to allowed and disallowed IP sets.

    Args:
        ip_address: The IP address to check
        ip_whitelist: Allowed IP addresses.
        ip_blacklist: Disallowed IP addresses.

    Returns:
        True if the IP address is in the blacklist and not in the whitelist.
    """
    if ip_address in ip_blacklist:
        if ip_whitelist is None or ip_address not in ip_whitelist:
            return True
    return False


class _IPBlacklistingResolver:
    """
    A proxy for reactor.nameResolver which only produces non-blacklisted IP
    addresses, preventing DNS rebinding attacks on URL preview.
    """

    def __init__(
        self,
        reactor: IReactorPluggableNameResolver,
        ip_whitelist: Optional[IPSet],
        ip_blacklist: IPSet,
    ):
        """
        Args:
            reactor: The twisted reactor.
            ip_whitelist: IP addresses to allow.
            ip_blacklist: IP addresses to disallow.
        """
        self._reactor = reactor
        self._ip_whitelist = ip_whitelist
        self._ip_blacklist = ip_blacklist

    def resolveHostName(
        self, recv: IResolutionReceiver, hostname: str, portNumber: int = 0
    ) -> IResolutionReceiver:
        addresses = []  # type: List[IAddress]

        def _callback() -> None:
            has_bad_ip = False
            for address in addresses:
                # We only expect IPv4 and IPv6 addresses since only A/AAAA lookups
                # should go through this path.
                if not isinstance(address, (IPv4Address, IPv6Address)):
                    continue

                ip_address = IPAddress(address.host)

                if check_against_blacklist(
                    ip_address, self._ip_whitelist, self._ip_blacklist
                ):
                    logger.info(
                        "Dropped %s from DNS resolution to %s due to blacklist"
                        % (ip_address, hostname)
                    )
                    has_bad_ip = True

            # if we have a blacklisted IP, we'd like to raise an error to block the
            # request, but all we can really do from here is claim that there were no
            # valid results.
            if not has_bad_ip:
                for address in addresses:
                    recv.addressResolved(address)
            recv.resolutionComplete()

        @provider(IResolutionReceiver)
        class EndpointReceiver:
            @staticmethod
            def resolutionBegan(resolutionInProgress: IHostResolution) -> None:
                recv.resolutionBegan(resolutionInProgress)

            @staticmethod
            def addressResolved(address: IAddress) -> None:
                addresses.append(address)

            @staticmethod
            def resolutionComplete() -> None:
                _callback()

        self._reactor.nameResolver.resolveHostName(
            EndpointReceiver, hostname, portNumber=portNumber
        )

        return recv


@implementer(IReactorPluggableNameResolver)
class BlacklistingReactorWrapper:
    """
    A Reactor wrapper which will prevent DNS resolution to blacklisted IP
    addresses, to prevent DNS rebinding.
    """

    def __init__(
        self,
        reactor: IReactorPluggableNameResolver,
        ip_whitelist: Optional[IPSet],
        ip_blacklist: IPSet,
    ):
        self._reactor = reactor

        # We need to use a DNS resolver which filters out blacklisted IP
        # addresses, to prevent DNS rebinding.
        self.nameResolver = _IPBlacklistingResolver(
            self._reactor, ip_whitelist, ip_blacklist
        )

    def __getattr__(self, attr: str) -> Any:
        # Passthrough to the real reactor except for the DNS resolver.
        return getattr(self._reactor, attr)
from sydent.http.blacklisting_reactor import BlacklistingReactorWrapper
from sydent.http.matrixfederationagent import MatrixFederationAgent
            BlacklistingReactorWrapper(
                reactor=self.sydent.reactor,
                ip_whitelist=sydent.ip_whitelist,
                ip_blacklist=sydent.ip_blacklist,
            ),
            BlacklistingReactorWrapper(
                reactor=self.sydent.reactor,
                ip_whitelist=sydent.ip_whitelist,
                ip_blacklist=sydent.ip_blacklist,
            ),
            ClientTLSOptionsFactory(sydent.cfg) if sydent.use_tls_for_federation else None,
from typing import Set
from sydent.util.ip_range import generate_ip_set, DEFAULT_IP_RANGE_BLACKLIST

        # Prevent outgoing requests from being sent to the following blacklisted
        # IP address CIDR ranges. If this option is not specified or empty then
        # it defaults to private IP address ranges.
        #
        # The blacklist applies to all outbound requests except replication
        # requests.
        #
        # (0.0.0.0 and :: are always blacklisted, whether or not they are
        # explicitly listed here, since they correspond to unroutable
        # addresses.)
        'ip.blacklist': '',

        # List of IP address CIDR ranges that should be allowed for outbound
        # requests. This is useful for specifying exceptions to wide-ranging
        # blacklisted target IP ranges.
        #
        # This whitelist overrides `ip.blacklist` and defaults to an empty
        # list.
        'ip.whitelist': '',
    def __init__(self, cfg, reactor=twisted.internet.reactor, use_tls_for_federation=True):
        self.use_tls_for_federation = use_tls_for_federation
        ip_blacklist = set_from_comma_sep_string(self.cfg.get("general", "ip.blacklist"))
        if not ip_blacklist:
            ip_blacklist = DEFAULT_IP_RANGE_BLACKLIST

        ip_whitelist = set_from_comma_sep_string(self.cfg.get("general", "ip.whitelist"))

        self.ip_blacklist = generate_ip_set(ip_blacklist)
        self.ip_whitelist = generate_ip_set(ip_whitelist)

def set_from_comma_sep_string(rawstr: str) -> Set[str]:
    if rawstr == '':
        return set()
    return {x.strip() for x in rawstr.split(',')}


#  Copyright 2021 The Matrix.org Foundation C.I.C.
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

import itertools
from typing import Iterable, Optional

from netaddr import AddrFormatError, IPNetwork, IPSet

# IP ranges that are considered private / unroutable / don't make sense.
DEFAULT_IP_RANGE_BLACKLIST = [
    # Localhost
    "127.0.0.0/8",
    # Private networks.
    "10.0.0.0/8",
    "172.16.0.0/12",
    "192.168.0.0/16",
    # Carrier grade NAT.
    "100.64.0.0/10",
    # Address registry.
    "192.0.0.0/24",
    # Link-local networks.
    "169.254.0.0/16",
    # Formerly used for 6to4 relay.
    "192.88.99.0/24",
    # Testing networks.
    "198.18.0.0/15",
    "192.0.2.0/24",
    "198.51.100.0/24",
    "203.0.113.0/24",
    # Multicast.
    "224.0.0.0/4",
    # Localhost
    "::1/128",
    # Link-local addresses.
    "fe80::/10",
    # Unique local addresses.
    "fc00::/7",
    # Testing networks.
    "2001:db8::/32",
    # Multicast.
    "ff00::/8",
    # Site-local addresses
    "fec0::/10",
]


def generate_ip_set(
    ip_addresses: Optional[Iterable[str]],
    extra_addresses: Optional[Iterable[str]] = None,
    config_path: Optional[Iterable[str]] = None,
) -> IPSet:
    """
    Generate an IPSet from a list of IP addresses or CIDRs.

    Additionally, for each IPv4 network in the list of IP addresses, also
    includes the corresponding IPv6 networks.

    This includes:

    * IPv4-Compatible IPv6 Address (see RFC 4291, section 2.5.5.1)
    * IPv4-Mapped IPv6 Address (see RFC 4291, section 2.5.5.2)
    * 6to4 Address (see RFC 3056, section 2)

    Args:
        ip_addresses: An iterable of IP addresses or CIDRs.
        extra_addresses: An iterable of IP addresses or CIDRs.
        config_path: The path in the configuration for error messages.

    Returns:
        A new IP set.
    """
    result = IPSet()
    for ip in itertools.chain(ip_addresses or (), extra_addresses or ()):
        try:
            network = IPNetwork(ip)
        except AddrFormatError as e:
            raise Exception(
                "Invalid IP range provided: %s." % (ip,), config_path
            ) from e
        result.add(network)

        # It is possible that these already exist in the set, but that's OK.
        if ":" not in str(network):
            result.add(IPNetwork(network).ipv6(ipv4_compatible=True))
            result.add(IPNetwork(network).ipv6(ipv4_compatible=False))
            result.add(_6to4(network))

    return result


def _6to4(network: IPNetwork) -> IPNetwork:
    """Convert an IPv4 network into a 6to4 IPv6 network per RFC 3056."""

    # 6to4 networks consist of:
    # * 2002 as the first 16 bits
    # * The first IPv4 address in the network hex-encoded as the next 32 bits
    # * The new prefix length needs to include the bits from the 2002 prefix.
    hex_network = hex(network.first)[2:]
    hex_network = ("0" * (8 - len(hex_network))) + hex_network
    return IPNetwork(
        "2002:%s:%s::/%d"
        % (
            hex_network[:4],
            hex_network[4:],
            16 + network.prefixlen,
        )
    )
#  Copyright 2021 The Matrix.org Foundation C.I.C.
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.


from mock import patch
from netaddr import IPSet
from twisted.internet import defer
from twisted.internet.error import DNSLookupError
from twisted.test.proto_helpers import StringTransport
from twisted.trial.unittest import TestCase
from twisted.web.client import Agent

from sydent.http.blacklisting_reactor import BlacklistingReactorWrapper
from sydent.http.srvresolver import Server
from tests.utils import make_request, make_sydent


class BlacklistingAgentTest(TestCase):
    def setUp(self):
        config = {
            "general": {
                "ip.blacklist": "5.0.0.0/8",
                "ip.whitelist": "5.1.1.1",
            },
        }

        self.sydent = make_sydent(test_config=config)

        self.reactor = self.sydent.reactor

        self.safe_domain, self.safe_ip = b"safe.test", b"1.2.3.4"
        self.unsafe_domain, self.unsafe_ip = b"danger.test", b"5.6.7.8"
        self.allowed_domain, self.allowed_ip = b"allowed.test", b"5.1.1.1"

        # Configure the reactor's DNS resolver.
        for (domain, ip) in (
            (self.safe_domain, self.safe_ip),
            (self.unsafe_domain, self.unsafe_ip),
            (self.allowed_domain, self.allowed_ip),
        ):
            self.reactor.lookups[domain.decode()] = ip.decode()
            self.reactor.lookups[ip.decode()] = ip.decode()

        self.ip_whitelist = self.sydent.ip_whitelist
        self.ip_blacklist = self.sydent.ip_blacklist

    def test_reactor(self):
        """Apply the blacklisting reactor and ensure it properly blocks
        connections to particular domains and IPs.
        """
        agent = Agent(
            BlacklistingReactorWrapper(
                self.reactor,
                ip_whitelist=self.ip_whitelist,
                ip_blacklist=self.ip_blacklist,
            ),
        )

        # The unsafe domains and IPs should be rejected.
        for domain in (self.unsafe_domain, self.unsafe_ip):
            self.failureResultOf(
                agent.request(b"GET", b"http://" + domain), DNSLookupError
            )

        self.reactor.tcpClients = []

        # The safe domains IPs should be accepted.
        for domain in (
            self.safe_domain,
            self.allowed_domain,
            self.safe_ip,
            self.allowed_ip,
        ):
            agent.request(b"GET", b"http://" + domain)

            # Grab the latest TCP connection.
            (
                host,
                port,
                client_factory,
                _timeout,
                _bindAddress,
            ) = self.reactor.tcpClients.pop()

    @patch("sydent.http.srvresolver.SrvResolver.resolve_service")
    def test_federation_client_allowed_ip(self, resolver):
        self.sydent.run()

        request, channel = make_request(
            self.sydent.reactor,
            "POST",
            "/_matrix/identity/v2/account/register",
            {
                "access_token": "foo",
                "expires_in": 300,
                "matrix_server_name": "example.com",
                "token_type": "Bearer",
            },
        )

        resolver.return_value = defer.succeed(
            [
                Server(
                    host=self.allowed_domain,
                    port=443,
                    priority=1,
                    weight=1,
                    expires=100,
                )
            ]
        )

        request.render(self.sydent.servlets.registerServlet)

        transport, protocol = self._get_http_request(
            self.allowed_ip.decode("ascii"), 443
        )

        self.assertRegex(
            transport.value(), b"^GET /_matrix/federation/v1/openid/userinfo"
        )
        self.assertRegex(transport.value(), b"Host: example.com")

        # Send it the HTTP response
        res_json = '{ "sub": "@test:example.com" }'.encode("ascii")
        protocol.dataReceived(
            b"HTTP/1.1 200 OK\r\n"
            b"Server: Fake\r\n"
            b"Content-Type: application/json\r\n"
            b"Content-Length: %i\r\n"
            b"\r\n"
            b"%s" % (len(res_json), res_json)
        )

        self.assertEqual(channel.code, 200)

    @patch("sydent.http.srvresolver.SrvResolver.resolve_service")
    def test_federation_client_safe_ip(self, resolver):
        self.sydent.run()

        request, channel = make_request(
            self.sydent.reactor,
            "POST",
            "/_matrix/identity/v2/account/register",
            {
                "access_token": "foo",
                "expires_in": 300,
                "matrix_server_name": "example.com",
                "token_type": "Bearer",
            },
        )

        resolver.return_value = defer.succeed(
            [
                Server(
                    host=self.safe_domain,
                    port=443,
                    priority=1,
                    weight=1,
                    expires=100,
                )
            ]
        )

        request.render(self.sydent.servlets.registerServlet)

        transport, protocol = self._get_http_request(self.safe_ip.decode("ascii"), 443)

        self.assertRegex(
            transport.value(), b"^GET /_matrix/federation/v1/openid/userinfo"
        )
        self.assertRegex(transport.value(), b"Host: example.com")

        # Send it the HTTP response
        res_json = '{ "sub": "@test:example.com" }'.encode("ascii")
        protocol.dataReceived(
            b"HTTP/1.1 200 OK\r\n"
            b"Server: Fake\r\n"
            b"Content-Type: application/json\r\n"
            b"Content-Length: %i\r\n"
            b"\r\n"
            b"%s" % (len(res_json), res_json)
        )

        self.assertEqual(channel.code, 200)

    @patch("sydent.http.srvresolver.SrvResolver.resolve_service")
    def test_federation_client_unsafe_ip(self, resolver):
        self.sydent.run()

        request, channel = make_request(
            self.sydent.reactor,
            "POST",
            "/_matrix/identity/v2/account/register",
            {
                "access_token": "foo",
                "expires_in": 300,
                "matrix_server_name": "example.com",
                "token_type": "Bearer",
            },
        )

        resolver.return_value = defer.succeed(
            [
                Server(
                    host=self.unsafe_domain,
                    port=443,
                    priority=1,
                    weight=1,
                    expires=100,
                )
            ]
        )

        request.render(self.sydent.servlets.registerServlet)

        self.assertNot(self.reactor.tcpClients)

        self.assertEqual(channel.code, 500)

    def _get_http_request(self, expected_host, expected_port):
        clients = self.reactor.tcpClients
        (host, port, factory, _timeout, _bindAddress) = clients[-1]
        self.assertEqual(host, expected_host)
        self.assertEqual(port, expected_port)

        # complete the connection and wire it up to a fake transport
        protocol = factory.buildProtocol(None)
        transport = StringTransport()
        protocol.makeConnection(transport)

        return transport, protocol
from typing import Dict
from zope.interface import implementer
from twisted.internet._resolver import SimpleResolverComplexifier
from twisted.internet.defer import fail, succeed
from twisted.internet.error import DNSLookupError
from twisted.internet.interfaces import (
    IHostnameResolver,
    IReactorPluggableNameResolver,
    IResolverSimple,
)

    if "db" not in test_config:
        test_config["db"] = {"db.file": ":memory:"}
        test_config["db"].setdefault("db.file", ":memory:")
    reactor = ResolvingMemoryReactorClock()
    return Sydent(reactor=reactor, cfg=parse_config_dict(test_config), use_tls_for_federation=False)

    if shorthand and not path.startswith(b"/_matrix"):
    log_format = "%(asctime)s - %(name)s - %(lineno)d - %(levelname)s" " - %(message)s"


@implementer(IReactorPluggableNameResolver)
class ResolvingMemoryReactorClock(MemoryReactorClock):
    """
    A MemoryReactorClock that supports name resolution.
    """

    def __init__(self):
        lookups = self.lookups = {}  # type: Dict[str, str]

        @implementer(IResolverSimple)
        class FakeResolver:
            def getHostByName(self, name, timeout=None):
                if name not in lookups:
                    return fail(DNSLookupError("OH NO: unknown %s" % (name,)))
                return succeed(lookups[name])

        self.nameResolver = SimpleResolverComplexifier(FakeResolver())
        super().__init__()

    def installNameResolver(self, resolver: IHostnameResolver) -> IHostnameResolver:
        raise NotImplementedError()