Repository: facelessuser/soupsieve

Issue:
Allow identifier tokens to start with `--` per the latest css-syntax-3 draft

Discussion:
facelessuser: Tests should be added to confirm the new functionality.

Patch:
diff --git a/docs/src/markdown/about/changelog.md b/docs/src/markdown/about/changelog.md
index bf628ca..0ce0aa2 100644
--- a/docs/src/markdown/about/changelog.md
+++ b/docs/src/markdown/about/changelog.md
@@ -3,7 +3,12 @@
 ## 1.8.0
 
 - **NEW**: Add custom selector support. (#92)
-- **FIX**: Small tweak to CSS identifier pattern to ensure it matches the CSS specification exactly.
+- **FIX**: Small tweak to CSS identifier pattern to ensure it matches the CSS specification exactly. Specifically, you
+can't have an identifier of only `-`.
+- **FIX**: CSS string patterns should allow escaping newlines to span strings across multiple lines.
+- **FIX**: Newline regular expression for CSS newlines should treat `\r\n` as a single character, especially in cases
+such as string escapes: `\\\r\n`.
+- **FIX**: Allow `--` as a valid identifier or identifier start.
 
 ## 1.7.3
 
diff --git a/soupsieve/css_parser.py b/soupsieve/css_parser.py
index 3308a0d..caa8767 100644
--- a/soupsieve/css_parser.py
+++ b/soupsieve/css_parser.py
@@ -85,27 +85,26 @@ PSEUDO_SUPPORTED = PSEUDO_SIMPLE | PSEUDO_SIMPLE_NO_MATCH | PSEUDO_COMPLEX | PSE
 
 # Sub-patterns parts
 # Whitespace
-WS = r'[ \t\r\n\f]'
+NEWLINE = r'(?:\r\n|(?!\r\n)[\n\f\r])'
+WS = r'(?:[ \t]|{})'.format(NEWLINE)
 # Comments
 COMMENTS = r'(?:/\*[^*]*\*+(?:[^/*][^*]*\*+)*/)'
 # Whitespace with comments included
 WSC = r'(?:{ws}|{comments})'.format(ws=WS, comments=COMMENTS)
 # CSS escapes
 CSS_ESCAPES = r'(?:\\[a-f0-9]{{1,6}}{ws}?|\\[^\r\n\f])'.format(ws=WS)
+CSS_STRING_ESCAPES = r'(?:\\[a-f0-9]{{1,6}}{ws}?|\\[^\r\n\f]|\\{nl})'.format(ws=WS, nl=NEWLINE)
 # CSS Identifier
 IDENTIFIER = r'''
-(?:-?(?:[^\x00-\x2f\x30-\x40\x5B-\x5E\x60\x7B-\x9f]|{esc})+
-(?:[^\x00-\x2c\x2e\x2f\x3A-\x40\x5B-\x5E\x60\x7B-\x9f]|{esc})*)
-'''.format(esc=CSS_ESCAPES)
-# CSS Identifier expanded to allow for custom start: `--`
-EXPANDED_IDENTIFIER = r'''
 (?:(?:-?(?:[^\x00-\x2f\x30-\x40\x5B-\x5E\x60\x7B-\x9f]|{esc})+|--)
 (?:[^\x00-\x2c\x2e\x2f\x3A-\x40\x5B-\x5E\x60\x7B-\x9f]|{esc})*)
 '''.format(esc=CSS_ESCAPES)
 # `nth` content
 NTH = r'(?:[-+])?(?:[0-9]+n?|n)(?:(?<=n){ws}*(?:[-+]){ws}*(?:[0-9]+))?'.format(ws=WSC)
 # Value: quoted string or identifier
-VALUE = r'''(?:"(?:\\.|[^\\"]+)*?"|'(?:\\.|[^\\']+)*?'|{ident}+)'''.format(ident=IDENTIFIER)
+VALUE = r'''
+(?:"(?:\\(?:.|{nl})|[^\\"\r\n\f]+)*?"|'(?:\\(?:.|{nl})|[^\\'\r\n\f]+)*?'|{ident}+)
+'''.format(nl=NEWLINE, ident=IDENTIFIER)
 # Attribute value comparison. `!=` is handled special as it is non-standard.
 ATTR = r'''
 (?:{ws}*(?P<cmp>[!~^|*$]?=){ws}*(?P<value>{value})(?:{ws}+(?P<case>[is]))?)?{ws}*\]
@@ -132,7 +131,7 @@ PAT_QUIRKS_ATTR = r'''
 # Pseudo class (`:pseudo-class`, `:pseudo-class(`)
 PAT_PSEUDO_CLASS = r'(?P<name>:{ident})(?P<open>\({ws}*)?'.format(ws=WSC, ident=IDENTIFIER)
 # Custom pseudo class (`:--custom-pseudo`)
-PAT_PSEUDO_CLASS_CUSTOM = r'(?P<name>:(?=--){ident})'.format(ident=EXPANDED_IDENTIFIER)
+PAT_PSEUDO_CLASS_CUSTOM = r'(?P<name>:(?=--){ident})'.format(ident=IDENTIFIER)
 # Closing pseudo group (`)`)
 PAT_PSEUDO_CLOSE = r'{ws}*\)'.format(ws=WSC)
 # Pseudo element (`::pseudo-element`)
@@ -154,13 +153,14 @@ PAT_PSEUDO_LANG = r':lang\({ws}*(?P<lang>{value}(?:{ws}*,{ws}*{value})*){ws}*\)'
 # Pseudo class direction (`:dir(ltr)`)
 PAT_PSEUDO_DIR = r':dir\({ws}*(?P<dir>ltr|rtl){ws}*\)'.format(ws=WSC)
 # Combining characters (`>`, `~`, ` `, `+`, `,`)
-PAT_COMBINE = r'{ws}*?(?P<relation>[,+>~]|[ \t\r\n\f](?![,+>~])){ws}*'.format(ws=WSC)
+PAT_COMBINE = r'{wsc}*?(?P<relation>[,+>~]|{ws}(?![,+>~])){wsc}*'.format(ws=WS, wsc=WSC)
 # Extra: Contains (`:contains(text)`)
 PAT_PSEUDO_CONTAINS = r':contains\({ws}*(?P<value>{value}){ws}*\)'.format(ws=WSC, value=VALUE)
 
 # Regular expressions
 # CSS escape pattern
 RE_CSS_ESC = re.compile(r'(?:(\\[a-f0-9]{{1,6}}{ws}?)|(\\[^\r\n\f]))'.format(ws=WSC), re.I)
+RE_CSS_STR_ESC = re.compile(r'(?:(\\[a-f0-9]{{1,6}}{ws}?)|(\\[^\r\n\f])|(\\{nl}))'.format(ws=WS, nl=NEWLINE), re.I)
 # Pattern to break up `nth` specifiers
 RE_NTH = re.compile(
     r'(?P<s1>[-+])?(?P<a>[0-9]+n?|n)(?:(?<=n){ws}*(?P<s2>[-+]){ws}*(?P<b>[0-9]+))?'.format(ws=WSC),
@@ -233,15 +233,31 @@ def _create_custom_selectors(custom, flags=0):
     return ct.CustomSelectors(**custom_selectors)
 
 
-def css_unescape(string):
-    """Unescape CSS value."""
+def css_unescape(content, string=False):
+    """
+    Unescape CSS value.
+
+    Strings allow for spanning the value on multiple strings by escaping a new line.
+    """
 
     def replace(m):
         """Replace with the appropriate substitute."""
 
         return util.uchr(int(m.group(1)[1:], 16)) if m.group(1) else m.group(2)[1:]
 
-    return RE_CSS_ESC.sub(replace, string)
+    def replace_string(m):
+        """Replace with the appropriate substitute for a string."""
+
+        if m.group(1):
+            value = util.uchr(int(m.group(1)[1:], 16))
+        elif m.group(2):
+            value = m.group(2)[1:]
+        else:
+            value = ''
+
+        return value
+
+    return RE_CSS_ESC.sub(replace, content) if not string else RE_CSS_STR_ESC.sub(replace_string, content)
 
 
 class SelectorPattern(object):
@@ -393,10 +409,10 @@ class CSSParser(object):
             flags = 0
 
         if op:
-            is_quoted = m.group('value').startswith(('"', "'")) and not quirks
-            value = css_unescape(
-                m.group('value')[1:-1] if is_quoted else m.group('value')
-            )
+            if m.group('value').startswith(('"', "'")) and not quirks:
+                value = css_unescape(m.group('value')[1:-1], True)
+            else:
+                value = css_unescape(m.group('value'))
         else:
             value = None
         if not op:
@@ -716,10 +732,9 @@ class CSSParser(object):
         selector = m.group(0)
         if selector.startswith('.'):
             sel.classes.append(css_unescape(selector[1:]))
-            has_selector = True
         else:
             sel.ids.append(css_unescape(selector[1:]))
-            has_selector = True
+        has_selector = True
         return has_selector
 
     def parse_pseudo_contains(self, sel, m, has_selector):
@@ -727,8 +742,9 @@ class CSSParser(object):
 
         content = m.group('value')
         if content.startswith(("'", '"')):
-            content = content[1:-1]
-        content = css_unescape(content)
+            content = css_unescape(content[1:-1], True)
+        else:
+            content = css_unescape(content)
         sel.contains.append(content)
         has_selector = True
         return has_selector
@@ -743,8 +759,9 @@ class CSSParser(object):
                 continue
             value = token.group('value')
             if value.startswith(('"', "'")):
-                value = value[1:-1]
-            parts = css_unescape(value).split('-')
+                parts = css_unescape(value[1:-1], True).split('-')
+            else:
+                parts = css_unescape(value).split('-')
 
             new_parts = []
             first = True


Test patch:
diff --git a/tests/test_extra/test_contains.py b/tests/test_extra/test_contains.py
index 7a2ac21..5b68ddb 100644
--- a/tests/test_extra/test_contains.py
+++ b/tests/test_extra/test_contains.py
@@ -46,6 +46,26 @@ class TestContains(util.TestCase):
             flags=util.HTML
         )
 
+    def test_contains_quoted_with_escaped_newline(self):
+        """Test contains quoted with escaped newline."""
+
+        self.assert_selector(
+            self.MARKUP,
+            'body :contains("Test\\\ning")',
+            ['1'],
+            flags=util.HTML
+        )
+
+    def test_contains_quoted_with_escaped_newline_with_carriage_return(self):
+        """Test contains quoted with escaped newline with carriage return."""
+
+        self.assert_selector(
+            self.MARKUP,
+            'body :contains("Test\\\r\ning")',
+            ['1'],
+            flags=util.HTML
+        )
+
     def test_contains_with_descendants(self):
         """Test that contains returns descendants as well as the top level that contain."""
 
diff --git a/tests/test_extra/test_custom.py b/tests/test_extra/test_custom.py
index 3b697ce..34b1700 100644
--- a/tests/test_extra/test_custom.py
+++ b/tests/test_extra/test_custom.py
@@ -47,6 +47,20 @@ class TestCustomSelectors(util.TestCase):
             flags=util.HTML
         )
 
+    def test_custom_selectors_exotic(self):
+        """Test custom selectors."""
+
+        custom_selectors = sv.Custom()
+        custom_selectors.append(":--", "h1, h2, h3, h4, h5, h6")
+
+        self.assert_selector(
+            self.MARKUP,
+            ':--',
+            ['1', '2'],
+            custom=custom_selectors,
+            flags=util.HTML
+        )
+
     def test_custom_dependency(self):
         """Test custom selector dependency on other custom selectors."""
 
diff --git a/tests/test_level1/test_type.py b/tests/test_level1/test_type.py
index b5de132..f7922f5 100644
--- a/tests/test_level1/test_type.py
+++ b/tests/test_level1/test_type.py
@@ -105,6 +105,7 @@ class TestType(util.TestCase):
         """Test invalid syntax."""
 
         self.assert_raises('div?', SelectorSyntaxError)
+        self.assert_raises('-', SelectorSyntaxError)
 
 
 class TestTypeQuirks(TestType):
diff --git a/tests/test_level2/test_attribute.py b/tests/test_level2/test_attribute.py
index c8c9036..5a919f7 100644
--- a/tests/test_level2/test_attribute.py
+++ b/tests/test_level2/test_attribute.py
@@ -140,6 +140,16 @@ class TestAttribute(util.TestCase):
             flags=util.HTML
         )
 
+    def test_attribute_escaped_newline(self):
+        """Test attribute with escaped new line in quoted string."""
+
+        self.assert_selector(
+            self.MARKUP,
+            '[id="pr\\\ne"]',
+            ["pre"],
+            flags=util.HTML
+        )
+
     def test_invalid_tag(self):
         """
         Test invalid tag.
diff --git a/tests/test_level4/test_lang.py b/tests/test_level4/test_lang.py
index 0306475..e2c2165 100644
--- a/tests/test_level4/test_lang.py
+++ b/tests/test_level4/test_lang.py
@@ -72,6 +72,17 @@ class TestLang(util.TestCase):
             flags=util.HTML
         )
 
+    def test_language_quoted_with_escaped_newline(self):
+        """Test language (quoted) with escaped new line."""
+
+        # Normal quoted
+        self.assert_selector(
+            self.MARKUP,
+            "p:lang('de-\\\nDE')",
+            ['1', '2', '3', '4', '5', '6'],
+            flags=util.HTML
+        )
+
     def test_wildcard_at_start_quoted(self):
         """Test language with wildcard at start (quoted)."""