Repository: facelessuser/soupsieve

Issue:
Handle escaped EOF
When parsing selectors, we don't really handle EOF like the spec says.  Does this really matter? 🤷‍♂️ 

When dealing with CSS parsing, the parser is *supposed* to translate an escaped EOF to  (U+FFFD). If the escaped EOF is in a string, it is supposed to be ignored.

So, according to spec, we are supposed to translate something like this `div\` (when the escape is containing the EOF) to something like this `div�`. When dealing with a string, something like this `"value\` would translate to `"value"`.

As far as strings go, none of our selectors can ever be valid if the string contains an escaped EOF.  This is because strings are either in `[attr="value"]` or something like `:lang("en-*")`.  If the string contained an escape EOF, the selector would be incomplete and throw an error anyways, so we really don't care about this edge case.

As far as identifiers are concerned, we could run into the case of `div\`. If this occurs, we should handle it in a sane manner. And according to the spec, this would translate to (U+FFFD).

Discussion:

Patch:
diff --git a/docs/src/dictionary/en-custom.txt b/docs/src/dictionary/en-custom.txt
index b739fd3..d4ca484 100644
--- a/docs/src/dictionary/en-custom.txt
+++ b/docs/src/dictionary/en-custom.txt
@@ -8,6 +8,7 @@ CSS's
 Changelog
 Combinators
 DOM
+EOF
 GitHub
 Hashable
 JQuery
diff --git a/docs/src/markdown/about/changelog.md b/docs/src/markdown/about/changelog.md
index c4102d4..a095b4b 100644
--- a/docs/src/markdown/about/changelog.md
+++ b/docs/src/markdown/about/changelog.md
@@ -5,8 +5,11 @@
 - **NEW**: Allow `:contans()` to accept a list of text to search for.
 - **FIX**: Don't install test files when installing the `soupsieve` package.
 - **FIX**: Improve efficiency of `:contains()` comparison.
-- **FIX**: Null characters should translate to the Unicode REPLACEMENT CHARACTER (`U+FFFD`) according to the spec. This
-applies to CSS escaped NULL characters as well.
+- **FIX**: Null characters should translate to the Unicode REPLACEMENT CHARACTER (`U+FFFD`) according to the
+specification. This applies to CSS escaped NULL characters as well.
+- **FIX**: Escaped EOF should translate to `U+FFFD` outside of CSS strings. In a string, they should just be ignored,
+but as there is no case where we could resolve such a string and still have a valid selector, string handling remains
+the same. (#128)
 
 ## 1.8.0
 
diff --git a/soupsieve/css_parser.py b/soupsieve/css_parser.py
index ba16fc9..88b3db4 100644
--- a/soupsieve/css_parser.py
+++ b/soupsieve/css_parser.py
@@ -94,8 +94,8 @@ COMMENTS = r'(?:/\*[^*]*\*+(?:[^/*][^*]*\*+)*/)'
 # Whitespace with comments included
 WSC = r'(?:{ws}|{comments})'.format(ws=WS, comments=COMMENTS)
 # CSS escapes
-CSS_ESCAPES = r'(?:\\[a-f0-9]{{1,6}}{ws}?|\\[^\r\n\f])'.format(ws=WS)
-CSS_STRING_ESCAPES = r'(?:\\[a-f0-9]{{1,6}}{ws}?|\\[^\r\n\f]|\\{nl})'.format(ws=WS, nl=NEWLINE)
+CSS_ESCAPES = r'(?:\\(?:[a-f0-9]{{1,6}}{ws}?|[^\r\n\f]|$))'.format(ws=WS)
+CSS_STRING_ESCAPES = r'(?:\\(?:[a-f0-9]{{1,6}}{ws}?|[^\r\n\f]|$|{nl}))'.format(ws=WS, nl=NEWLINE)
 # CSS Identifier
 IDENTIFIER = r'''
 (?:(?:-?(?:[^\x00-\x2f\x30-\x40\x5B-\x5E\x60\x7B-\x9f]|{esc})+|--)
@@ -161,8 +161,10 @@ PAT_PSEUDO_CONTAINS = r':contains\({ws}*(?P<values>{value}(?:{ws}*,{ws}*{value})
 
 # Regular expressions
 # CSS escape pattern
-RE_CSS_ESC = re.compile(r'(?:(\\[a-f0-9]{{1,6}}{ws}?)|(\\[^\r\n\f]))'.format(ws=WSC), re.I)
-RE_CSS_STR_ESC = re.compile(r'(?:(\\[a-f0-9]{{1,6}}{ws}?)|(\\[^\r\n\f])|(\\{nl}))'.format(ws=WS, nl=NEWLINE), re.I)
+RE_CSS_ESC = re.compile(r'(?:(\\[a-f0-9]{{1,6}}{ws}?)|(\\[^\r\n\f])|(\\$))'.format(ws=WSC), re.I)
+RE_CSS_STR_ESC = re.compile(
+    r'(?:(\\[a-f0-9]{{1,6}}{ws}?)|(\\[^\r\n\f])|(\\$)|(\\{nl}))'.format(ws=WS, nl=NEWLINE), re.I
+)
 # Pattern to break up `nth` specifiers
 RE_NTH = re.compile(
     r'(?P<s1>[-+])?(?P<a>[0-9]+n?|n)(?:(?<=n){ws}*(?P<s2>[-+]){ws}*(?P<b>[0-9]+))?'.format(ws=WSC),
@@ -249,6 +251,8 @@ def css_unescape(content, string=False):
             value = util.uchr(codepoint)
         elif m.group(2):
             value = m.group(2)[1:]
+        elif m.group(3):
+            value = '\ufffd'
         else:
             value = ''
 


Test patch:
diff --git a/tests/test_level1/test_class.py b/tests/test_level1/test_class.py
index 105455a..6e0e581 100644
--- a/tests/test_level1/test_class.py
+++ b/tests/test_level1/test_class.py
@@ -20,7 +20,7 @@ class TestClass(util.TestCase):
     # to ensure consistent behavior in our tests across parsers.
     MARKUP_NULL = """
     <div>
-    <p>Some text <span id="1" class="\ufffdfoo"> in a paragraph</span>.
+    <p>Some text <span id="1" class="foo\ufffd"> in a paragraph</span>.
     <a id="2" class="\ufffdbar" href="http://google.com">Link</a>
     </p>
     </div>
@@ -56,6 +56,16 @@ class TestClass(util.TestCase):
             flags=util.HTML
         )
 
+    def test_type_and_class_escaped_eof(self):
+        """Test type and class with an escaped EOF."""
+
+        self.assert_selector(
+            self.MARKUP_NULL,
+            "span.foo\\",
+            ["1"],
+            flags=util.HTML
+        )
+
     def test_malformed_class(self):
         """Test malformed class."""