CVE-2016-9910 (Python, published 2017-02-22T16:59Z)
Description: value: The serializer in html5lib before 0.99999999 might allow remote attackers to conduct cross-site scripting (XSS) attacks by leveraging mishandling of special characters in attribute values, a different vulnerability than CVE-2016-9909.
Severity: MEDIUM (CVSS v3 base score: 6.1)
Weakness: CWE-79 \u2014 Improper Neutralization of Input During Web Page Generation ('Cross-site Scripting')
The product does not neutralize or incorrectly neutralizes user-controllable input before it is placed in output that is used as a web page that is served to other users.
Repository: https://github.com/html5lib/html5lib-python
Files changed:
CHANGES.rst
html5lib/serializer/htmlserializer.py
html5lib/tests/serializer-testdata/core.test
html5lib/tests/serializer-testdata/options.test
html5lib/tests/test_serializer.py
Commit message: Fix #11, #12: quote attributes that need escaping in legacy browsers

These are mostly out of the market now, so this isn't massively
needed any more; nevertheless, avoiding XSS as much as possible is
inevitably desirable.

This alters the API so that quote_attr_values is now a ternary
setting, choosing between legacy-safe behaviour, spec behaviour, and
always quoting.


Vulnerable code:
quoteAttributeSpec = re.compile("[" + spaceCharacters + "\"'=<>`]")
    quote_attr_values = False
        quote_attr_values=True|False
          per HTML5 parsing rules.
                        if self.quote_attr_values:
                            quote_attr = len(v) == 0 or quoteAttributeSpec.search(v)
                "<span title=foo\u000bbar>"
            "description": "quote_attr_values=true",
                "quote_attr_values": true
            "description": "quote_attr_values=true with irrelevant",
                "quote_attr_values": true

Fixed code:
* **Fix #11, avoiding the XSS bug potentially caused by serializer
  allowing attribute values to be escaped out of in old browser versions,
  changing the quote_attr_values option on serializer to take one of
  three values, "always" (the old True value), "legacy" (the new option,
  and the new default), and "spec" (the old False value, and the old
  default).**

quoteAttributeSpecChars = spaceCharacters + "\"'=<>`"
quoteAttributeSpec = re.compile("[" + quoteAttributeSpecChars + "]")
quoteAttributeLegacy = re.compile("[" + quoteAttributeSpecChars +
                                  "\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n"
                                  "\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15"
                                  "\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
                                  "\x20\x2f\x60\xa0\u1680\u180e\u180f\u2000"
                                  "\u2001\u2002\u2003\u2004\u2005\u2006\u2007"
                                  "\u2008\u2009\u200a\u2028\u2029\u202f\u205f"
                                  "\u3000]")
    quote_attr_values = "legacy"  # be secure by default
        quote_attr_values="legacy"|"spec"|"always"
          per legacy browser behaviour, when required by the standard, or always.
                        if self.quote_attr_values == "always" or len(v) == 0:
                        elif self.quote_attr_values == "spec":
                            quote_attr = quoteAttributeSpec.search(v) is not None
                        elif self.quote_attr_values == "legacy":
                            quote_attr = quoteAttributeLegacy.search(v) is not None
                            raise ValueError("quote_attr_values must be one of: "
                                             "'always', 'spec', or 'legacy'")
                "<span title=\"foo\u000bbar\">"
            "description": "quote_attr_values='always'",
                "quote_attr_values": "always"
            "description": "quote_attr_values='always' with irrelevant",
                "quote_attr_values": "always"
            }
        },
        {
            "expected": [
                "<div class=\"foo\">"
            ],
            "input": [
                [
                    "StartTag",
                    "http://www.w3.org/1999/xhtml",
                    "div",
                    [
                        {
                            "namespace": null,
                            "name": "class",
                            "value": "foo"
                        }
                    ]
                ]
            ],
            "description": "non-minimized quote_attr_values='always'",
            "options": {
                "quote_attr_values": "always"
            }
        },
        {
            "expected": [
                "<div class=foo>"
            ],
            "input": [
                [
                    "StartTag",
                    "http://www.w3.org/1999/xhtml",
                    "div",
                    [
                        {
                            "namespace": null,
                            "name": "class",
                            "value": "foo"
                        }
                    ]
                ]
            ],
            "description": "non-minimized quote_attr_values='legacy'",
            "options": {
                "quote_attr_values": "legacy"
            }
        },
        {
            "expected": [
                "<div class=foo>"
            ],
            "input": [
                [
                    "StartTag",
                    "http://www.w3.org/1999/xhtml",
                    "div",
                    [
                        {
                            "namespace": null,
                            "name": "class",
                            "value": "foo"
                        }
                    ]
                ]
            ],
            "description": "non-minimized quote_attr_values='spec'",
            "options": {
                "quote_attr_values": "spec"
@pytest.mark.parametrize("c", list("\t\n\u000C\x20\r\"'=<>`"))
def testSpecQuoteAttribute(c):
    input_ = [["StartTag", "http://www.w3.org/1999/xhtml", "span",
               [{"namespace": None, "name": "foo", "value": c}]]]
    if c == '"':
        output_ = ["<span foo='%s'>" % c]
    else:
        output_ = ['<span foo="%s">' % c]
    options_ = {"quote_attr_values": "spec"}
    runSerializerTest(input_, output_, options_)


@pytest.mark.parametrize("c", list("\t\n\u000C\x20\r\"'=<>`"
                                   "\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n"
                                   "\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15"
                                   "\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f"
                                   "\x20\x2f\x60\xa0\u1680\u180e\u180f\u2000"
                                   "\u2001\u2002\u2003\u2004\u2005\u2006\u2007"
                                   "\u2008\u2009\u200a\u2028\u2029\u202f\u205f"
                                   "\u3000"))
def testLegacyQuoteAttribute(c):
    input_ = [["StartTag", "http://www.w3.org/1999/xhtml", "span",
               [{"namespace": None, "name": "foo", "value": c}]]]
    if c == '"':
        output_ = ["<span foo='%s'>" % c]
    else:
        output_ = ['<span foo="%s">' % c]
    options_ = {"quote_attr_values": "legacy"}
    runSerializerTest(input_, output_, options_)