Repository: hylang/hy

Issue:
Invalid string escape sequences become literal backslashes
Python's traditional behavior given something like `"a\p"` is to silently treat it as `"a\\p"` (or equivalently, `r"a\p"`), because there is no escape sequence `\p`. Recent Pythons will produce a `DeprecationWarning` for this, although you may have to opt in to the warning. Hy currently treats  `"a\p"` as `"a\\p"` with no warning and no error.

Since we're still not at 1.0, I think we might as well jump headfirst into the future and treat invalid escape sequences as syntax errors, regardless of the underlying Python version. The literal-backslash fallback isn't useful and is liable to get the programmer into trouble.


Discussion:
Oh neat! I guess this brings up the question of how we want to handle versiones incremental Python changes going forward. Do we match their cadence? I.e. make it a deprecation warning on those Python versions and a syntax error on the others?
I don't know. I guess we don't have to think about it too hard till after we're at 1.0, and it's hard to foresee new Python changes that would suggest a corresponding Hy deprecation as in this case. In the past, Hy has smoothed over some of the differences between Python versions when it's easy to do so, and I think that can be helpful.
True enough. We are getting pretty close though and this has me thinking about how we telegraph breaking changes in future builds since we can't be breaking stuff willy nilly past 1.0 anymore. We can have that process discussion when we get there though. Also, this has me realizing that hy doesn't have an option to enable deprecation warnings at all. You'd usually pass `-Wd` to python, but hy has no corresponding arg to pass that through 🤷‍♀️ 

Resolving patch:
diff --git a/hy/lex/hy_reader.py b/hy/lex/hy_reader.py
--- a/hy/lex/hy_reader.py
+++ b/hy/lex/hy_reader.py
@@ -250,6 +250,14 @@ def keyword(self, _):
 
     @reader_for('"')
     def prefixed_string(self, _, prefix=""):
+        prefix_chars = set(prefix)
+        if (
+            len(prefix_chars) != len(prefix)
+            or prefix_chars - set("bfr")
+            or set("bf") <= prefix_chars
+        ):
+            raise LexException.from_reader(f"invalid string prefix {prefix!r}", self)
+
         escaping = False
 
         def quote_closing(c):
@@ -259,6 +267,15 @@ def quote_closing(c):
                 return 0
             if c == '"' and not escaping:
                 return 1
+            if (
+                escaping
+                and "r" not in prefix
+                and
+                # https://docs.python.org/3/reference/lexical_analysis.html#string-and-bytes-literals
+                c
+                not in ("\n\r\\'\"abfnrtv01234567x" + ("" if "b" in prefix else "NuU"))
+            ):
+                raise LexException.from_reader("invalid escape sequence \\" + c, self)
             escaping = False
             return 0
 
@@ -396,8 +413,10 @@ def bracketed_string(self, _):
         delim = "".join(delim)
         is_fstring = delim == "f" or delim.startswith("f-")
 
-        # discard single initial newline, if any
-        self.peek_and_getc("\n")
+        # discard single initial newline, if any, accounting for all
+        # three styles of newline
+        self.peek_and_getc("\x0d")
+        self.peek_and_getc("\x0a")
 
         index = -1
 
@@ -447,7 +466,8 @@ def read_chars_until(self, closing, prefix, is_fstring):
                     # remove "{" from end of string component
                     s.pop()
                     break
-        res = "".join(s)
+        res = "".join(s).replace("\x0d\x0a", "\x0a").replace("\x0d", "\x0a")
+
         if prefix is not None:
             res = eval(f'{prefix}"""{res}"""')
         if is_fstring:


Test patch:
diff --git a/tests/compilers/test_ast.py b/tests/compilers/test_ast.py
--- a/tests/compilers/test_ast.py
+++ b/tests/compilers/test_ast.py
@@ -499,11 +499,12 @@ def test_ast_bracket_string():
 ]X]''') == '\n\nOnly one leading newline should be removed.\n')
 
 
-@pytest.mark.xfail
 def test_literal_newlines():
     # https://github.com/hylang/hy/issues/2239
     assert s('"\r\nhello\r\nworld"') == "\nhello\nworld"
     assert s('r"\r\nhello\r\nworld"') == "\nhello\nworld"
+    assert s('b"\r\nhello\r\nworld"') == b"\nhello\nworld"
+    assert s('br"\r\nhello\r\nworld"') == b"\nhello\nworld"
     assert s("#[[\r\nhello\r\nworld]]") == "hello\nworld"
     assert s("#[[\rhello\rworld]]") == "hello\nworld"
 
diff --git a/tests/test_lex.py b/tests/test_lex.py
--- a/tests/test_lex.py
+++ b/tests/test_lex.py
@@ -8,10 +8,12 @@
 from hy.lex import read_many
 from hy.lex.exceptions import LexException, PrematureEndOfInput
 from hy.models import (
+    Bytes,
     Complex,
     Dict,
     Expression,
     Float,
+    FString,
     Integer,
     Keyword,
     List,
@@ -437,13 +439,39 @@ def test_nospace():
     assert entry.end_column == 13
 
 
-def test_escapes():
-    """Ensure we can escape things"""
-    entry = tokenize(r"""(foo "foo\n")""")[0]
-    assert entry[1] == String("foo\n")
+def test_string_prefixes():
+    s = lambda x: tokenize(x)[0]
+
+    assert s(r'b"hello"') == Bytes(b"hello")
+    assert s(r'rb"hello"') == Bytes(b"hello")
+    assert s(r'fr"hello"') == FString([String("hello")])
+
+    for bad in list("zRBFu") + ["bf", "rr", "rbr"]:
+        with lexe():
+            s(bad + '"hello"')
+
 
-    entry = tokenize(r"""(foo r"foo\s")""")[0]
-    assert entry[1] == String(r"foo\s")
+def test_escapes():
+    s = lambda x: tokenize(x)[0]
+
+    # A valid escape sequence
+    assert s(r'"foo\x5a"') == String("fooZ")
+    assert s(r'b"foo\x5a"') == Bytes(b"fooZ")
+    # In a raw string
+    assert s(r'r"foo\x5a"') == String("foo\\x5a")
+    assert s(r'rb"foo\x5a"') == Bytes(b"foo\\x5a")
+    # An invalid escape sequence
+    with lexe():
+        s(r'"foo\s"')
+    with lexe():
+        s(r'b"foo\s"')
+    # In a raw string
+    assert s(r'r"foo\s"') == String("foo\\s")
+    assert s(r'rb"foo\s"') == Bytes(b"foo\\s")
+    # An escape sequence that's valid in strings, but not bytes.
+    assert s(r'"foo\u005a"') == String("fooZ")
+    with lexe():
+        s(r'b"foo\u005a"')
 
 
 def test_unicode_escapes():