Repository: falconry/falcon

Issue:
Incorrect handling of commas inside a Forwarded header
To implement [`Request.forwarded`](http://falcon.readthedocs.io/en/stable/api/request_and_response.html#Request.forwarded), Falcon splits the incoming Forwarded header on commas. Because of this, it cannot correctly handle forwarded-elements that contain a comma inside a quoted-string.

To reproduce, run the following `fwd_demo.py`:

```
import falcon

class ForwardedResource(object):
    def on_get(self, req, resp):
        resp.body = req.forwarded[-1].src

app = falcon.API()
app.add_route('/fwd', ForwardedResource())
```

under Gunicorn:

```
$ gunicorn fwd_demo:app
```

and send requests like this:

```
curl -s localhost:8000/fwd -H 'Forwarded: for=1.2.3.4;some="extra,info"'
```

The expected response is `1.2.3.4`, but the actual response is empty.

A more correct solution would be to parse the Forwarded header carefully with regard to quoted-strings. However, this is complicated by the fact that the Forwarded header may be partly controlled by an external attacker, who might submit a request with:

```
Forwarded: for=_trusted_addr;by="
```

Unless a reverse proxy eagerly validates and fixes/strips such a header, appending another forwarded-element might give:

```
Forwarded: for=_trusted_addr;by=", for=_real_addr
```

In this case, `req.forwarded[-1].src` ought to be `_real_addr`, not `_trusted_addr`.

See [aiohttp’s implementation of Forwarded](https://github.com/aio-libs/aiohttp/blob/1609acd5802364fa3bd11adfb34a4b2b0d07fc42/aiohttp/web_request.py#L188-L246) for an example of how to deal with this correctly (I hope).

“`Request.forwarded` should not be trusted anyway” would be a poor solution, I think. As long as a reverse proxy is correctly configured to append a header, that header may be and often is trusted by the application for important purposes such as rate limiting.



Discussion:
Hi @vfaronov, good catch. Let's schedule this issue for the next release.

Resolving patch:
diff --git a/falcon/forwarded.py b/falcon/forwarded.py
new file mode 100644
--- /dev/null
+++ b/falcon/forwarded.py
@@ -0,0 +1,184 @@
+# Based on code from the aiohttp project, Copyright 2013-2017 by Nikolay Kim and
+# Andrew Svetlov, with modifications for the Falcon project by Kurt Griffiths.
+#
+# See also:
+#
+#   https://github.com/aio-libs/aiohttp/blob/master/aiohttp/web_request.py
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import re
+import string
+
+from falcon.util.uri import unquote_string
+
+
+# '-' at the end to prevent interpretation as range in a char class
+_TCHAR = string.digits + string.ascii_letters + r"!#$%&'*+.^_`|~-"
+
+_TOKEN = r'[{tchar}]+'.format(tchar=_TCHAR)
+
+# qdtext includes 0x5C to escape 0x5D ('\]')
+# qdtext excludes obs-text (because obsoleted, and encoding not specified)
+_QDTEXT = r'[{0}]'.format(
+    r''.join(chr(c) for c in (0x09, 0x20, 0x21) + tuple(range(0x23, 0x7F))))
+
+_QUOTED_PAIR = r'\\[\t !-~]'
+
+_QUOTED_STRING = r'"(?:{quoted_pair}|{qdtext})*"'.format(
+    qdtext=_QDTEXT, quoted_pair=_QUOTED_PAIR)
+
+_FORWARDED_PAIR = (
+    r'({token})=({token}|{quoted_string})'.format(
+        token=_TOKEN,
+        quoted_string=_QUOTED_STRING))
+
+# same pattern as _QUOTED_PAIR but contains a capture group
+_QUOTED_PAIR_REPLACE_RE = re.compile(r'\\([\t !-~])')
+
+_FORWARDED_PAIR_RE = re.compile(_FORWARDED_PAIR)
+
+
+class Forwarded(object):
+    """Represents a parsed Forwarded header.
+
+    (See also: RFC 7239, Section 4)
+
+    Attributes:
+        src (str): The value of the "for" parameter, or
+            ``None`` if the parameter is absent. Identifies the
+            node making the request to the proxy.
+        dest (str): The value of the "by" parameter, or
+            ``None`` if the parameter is absent. Identifies the
+            client-facing interface of the proxy.
+        host (str): The value of the "host" parameter, or
+            ``None`` if the parameter is absent. Provides the host
+            request header field as received by the proxy.
+        scheme (str): The value of the "proto" parameter, or
+            ``None`` if the parameter is absent. Indicates the
+            protocol that was used to make the request to
+            the proxy.
+    """
+
+    # NOTE(kgriffs): Use "client" since "for" is a keyword, and
+    # "scheme" instead of "proto" to be consistent with the
+    # falcon.Request interface.
+    __slots__ = ('src', 'dest', 'host', 'scheme')
+
+    def __init__(self):
+        self.src = None
+        self.dest = None
+        self.host = None
+        self.scheme = None
+
+
+def _parse_forwarded_header(forwarded):
+    """Parses the value of a Forwarded header.
+
+    Makes an effort to parse Forwarded headers as specified by RFC 7239:
+
+    - It checks that every value has valid syntax in general as specified
+      in section 4: either a 'token' or a 'quoted-string'.
+    - It un-escapes found escape sequences.
+    - It does NOT validate 'by' and 'for' contents as specified in section
+      6.
+    - It does NOT validate 'host' contents (Host ABNF).
+    - It does NOT validate 'proto' contents for valid URI scheme names.
+
+    Arguments:
+        forwarded (str): Value of a Forwarded header
+
+    Returns:
+        list: Sequence of Forwarded instances, representing each forwarded-element
+        in the header, in the same order as they appeared in the header.
+    """
+
+    elements = []
+
+    pos = 0
+    end = len(forwarded)
+    need_separator = False
+    parsed_element = None
+
+    while 0 <= pos < end:
+        match = _FORWARDED_PAIR_RE.match(forwarded, pos)
+
+        if match is not None:           # got a valid forwarded-pair
+            if need_separator:
+                # bad syntax here, skip to next comma
+                pos = forwarded.find(',', pos)
+
+            else:
+                pos += len(match.group(0))
+                need_separator = True
+
+                name, value = match.groups()
+
+                # NOTE(kgriffs): According to RFC 7239, parameter
+                # names are case-insensitive.
+                name = name.lower()
+
+                if value[0] == '"':
+                    value = unquote_string(value)
+
+                # NOTE(kgriffs): If this is the first pair we've encountered
+                # for this forwarded-element, initialize a new object.
+                if not parsed_element:
+                    parsed_element = Forwarded()
+
+                if name == 'by':
+                    parsed_element.dest = value
+                elif name == 'for':
+                    parsed_element.src = value
+                elif name == 'host':
+                    parsed_element.host = value
+                elif name == 'proto':
+                    # NOTE(kgriffs): RFC 7239 only requires that
+                    # the "proto" value conform to the Host ABNF
+                    # described in RFC 7230. The Host ABNF, in turn,
+                    # does not require that the scheme be in any
+                    # particular case, so we normalize it here to be
+                    # consistent with the WSGI spec that *does*
+                    # require the value of 'wsgi.url_scheme' to be
+                    # either 'http' or 'https' (case-sensitive).
+                    parsed_element.scheme = value.lower()
+
+        elif forwarded[pos] == ',':      # next forwarded-element
+            need_separator = False
+            pos += 1
+
+            # NOTE(kgriffs): It's possible that we arrive here without a
+            # parsed element if the header is malformed.
+            if parsed_element:
+                elements.append(parsed_element)
+                parsed_element = None
+
+        elif forwarded[pos] == ';':      # next forwarded-pair
+            need_separator = False
+            pos += 1
+
+        elif forwarded[pos] in ' \t':
+            # Allow whitespace even between forwarded-pairs, though
+            # RFC 7239 doesn't. This simplifies code and is in line
+            # with Postel's law.
+            pos += 1
+
+        else:
+            # bad syntax here, skip to next comma
+            pos = forwarded.find(',', pos)
+
+    # NOTE(kgriffs): Add the last forwarded-element, if any
+    if parsed_element:
+        elements.append(parsed_element)
+
+    return elements
diff --git a/falcon/request.py b/falcon/request.py
--- a/falcon/request.py
+++ b/falcon/request.py
@@ -37,9 +37,11 @@
 from falcon import errors
 from falcon import request_helpers as helpers
 from falcon import util
+from falcon.forwarded import _parse_forwarded_header
+from falcon.forwarded import Forwarded  # NOQA
 from falcon.media import Handlers
 from falcon.util import json
-from falcon.util.uri import parse_host, parse_query_string, unquote_string
+from falcon.util.uri import parse_host, parse_query_string
 
 # NOTE(tbug): In some cases, http_cookies is not a module
 # but a dict-like structure. This fixes that issue.
@@ -231,7 +233,8 @@ class Request(object):
 
         forwarded (list): Value of the Forwarded header, as a parsed list
             of :class:`falcon.Forwarded` objects, or ``None`` if the header
-            is missing.
+            is missing. If the header value is malformed, Falcon will
+            make a best effort to parse what it can.
 
             (See also: RFC 7239, Section 4)
         date (datetime): Value of the Date header, converted to a
@@ -549,49 +552,7 @@ def forwarded(self):
             except KeyError:
                 return None
 
-            parsed_elements = []
-
-            for element in forwarded.split(','):
-                parsed_element = Forwarded()
-
-                # NOTE(kgriffs): Calling strip() is necessary here since
-                # "an HTTP list allows white spaces to occur between the
-                # identifiers".
-
-                # (See also: RFC 7239, Section 7.1)
-                for param in element.strip().split(';'):
-                    # PERF(kgriffs): partition() is faster than split().
-                    name, __, value = param.partition('=')
-                    if not value:
-                        # NOTE(kgriffs): The '=' separator was not found or
-                        # the value was missing. Ignore this malformed
-                        # param.
-                        continue
-
-                    # NOTE(kgriffs): According to RFC 7239, parameter
-                    # names are case-insensitive.
-                    name = name.lower()
-                    value = unquote_string(value)
-                    if name == 'by':
-                        parsed_element.dest = value
-                    elif name == 'for':
-                        parsed_element.src = value
-                    elif name == 'host':
-                        parsed_element.host = value
-                    elif name == 'proto':
-                        # NOTE(kgriffs): RFC 7239 only requires that
-                        # the "proto" value conform to the Host ABNF
-                        # described in RFC 7230. The Host ABNF, in turn,
-                        # does not require that the scheme be in any
-                        # particular case, so we normalize it here to be
-                        # consistent with the WSGI spec that *does*
-                        # require the value of 'wsgi.url_scheme' to be
-                        # either 'http' or 'https' (case-sensitive).
-                        parsed_element.scheme = value.lower()
-
-                parsed_elements.append(parsed_element)
-
-            self._cached_forwarded = parsed_elements
+            self._cached_forwarded = _parse_forwarded_header(forwarded)
 
         return self._cached_forwarded
 
@@ -1769,36 +1730,3 @@ def __init__(self):
         self.strip_url_path_trailing_slash = True
         self.default_media_type = DEFAULT_MEDIA_TYPE
         self.media_handlers = Handlers()
-
-
-class Forwarded(object):
-    """Represents a parsed Forwarded header.
-
-    (See also: RFC 7239, Section 4)
-
-    Attributes:
-        src (str): The value of the "for" parameter, or
-            ``None`` if the parameter is absent. Identifies the
-            node making the request to the proxy.
-        dest (str): The value of the "by" parameter, or
-            ``None`` if the parameter is absent. Identifies the
-            client-facing interface of the proxy.
-        host (str): The value of the "host" parameter, or
-            ``None`` if the parameter is absent. Provides the host
-            request header field as received by the proxy.
-        scheme (str): The value of the "proto" parameter, or
-            ``None`` if the parameter is absent. Indicates the
-            protocol that was used to make the request to
-            the proxy.
-    """
-
-    # NOTE(kgriffs): Use "client" since "for" is a keyword, and
-    # "scheme" instead of "proto" to be consistent with the
-    # falcon.Request interface.
-    __slots__ = ('src', 'dest', 'host', 'scheme')
-
-    def __init__(self):
-        self.src = None
-        self.dest = None
-        self.host = None
-        self.scheme = None


Test patch:
diff --git a/tests/test_request_forwarded.py b/tests/test_request_forwarded.py
--- a/tests/test_request_forwarded.py
+++ b/tests/test_request_forwarded.py
@@ -1,3 +1,5 @@
+import pytest
+
 from falcon.request import Request
 import falcon.testing as testing
 
@@ -120,3 +122,40 @@ def test_forwarded_missing_first_hop_host():
     assert req.forwarded_uri == req.uri
     assert req.forwarded_uri == 'http://suchproxy02.suchtesting.com/doge/languages'
     assert req.forwarded_prefix == 'http://suchproxy02.suchtesting.com/doge'
+
+
+def test_forwarded_quote_escaping():
+    req = Request(testing.create_environ(
+        host='suchproxy02.suchtesting.com',
+        path='/languages',
+        app='doge',
+        headers={
+            'Forwarded': 'for="1\\.2\\.3\\.4";some="extra,\\"info\\""'
+        }
+    ))
+
+    assert req.forwarded[0].host is None
+    assert req.forwarded[0].src == '1.2.3.4'
+
+
+@pytest.mark.parametrize('forwarded, expected_dest', [
+    ('for=1.2.3.4;by="', None),
+    ('for=1.2.3.4;by=4\\.3.2.1thing=blah', '4'),
+    ('for=1.2.3.4;by="\\4.3.2.1"thing=blah', '4.3.2.1'),
+    ('for=1.2.3.4;by="4.3.2.\\1"thing="blah"', '4.3.2.1'),
+    ('for=1.2.3.4;by="4.3.\\2\\.1" thing="blah"', '4.3.2.1'),
+])
+def test_escape_malformed_requests(forwarded, expected_dest):
+
+    req = Request(testing.create_environ(
+        host='suchproxy02.suchtesting.com',
+        path='/languages',
+        app='doge',
+        headers={
+            'Forwarded': forwarded
+        }
+    ))
+
+    assert len(req.forwarded) == 1
+    assert req.forwarded[0].src == '1.2.3.4'
+    assert req.forwarded[0].dest == expected_dest
diff --git a/tests/test_utils.py b/tests/test_utils.py
--- a/tests/test_utils.py
+++ b/tests/test_utils.py
@@ -229,6 +229,13 @@ def test_prop_uri_decode_models_stdlib_unquote_plus(self):
             actual = uri.decode(case)
             assert expect == actual
 
+    def test_unquote_string(self):
+        assert uri.unquote_string('v') == 'v'
+        assert uri.unquote_string('not-quoted') == 'not-quoted'
+        assert uri.unquote_string('partial-quoted"') == 'partial-quoted"'
+        assert uri.unquote_string('"partial-quoted') == '"partial-quoted'
+        assert uri.unquote_string('"partial-quoted"') == 'partial-quoted'
+
     def test_parse_query_string(self):
         query_strinq = (
             'a=http%3A%2F%2Ffalconframework.org%3Ftest%3D1'