Repository: hylang/hy

Issue:
Removing u""
After looking through #1226, I realized the `u` prefix is discarded. Why is this even allowed? Seems like it'd be potential cause for confusion for a Hy newbie. IMO, it would be better if it printed an error saying that all Hy strings are already Unicode.

Thoughts?

Discussion:
allison-casey: Looks good from my cursory glance. I'd only ask that you link to the Python page about unrecognized escape sequences being deprecated/syntax errors. I'll try and give this a more thorough review today/tomorrow
Kodiologist: Sure, done.

Patch:
diff --git a/NEWS.rst b/NEWS.rst
index 2fa47ad7..6d562280 100644
--- a/NEWS.rst
+++ b/NEWS.rst
@@ -25,10 +25,15 @@ Breaking Changes
 * `defmacro` once again requires the macro name as a symbol, not
   a string literal.
 * The parser has been completely rewritten. It is mostly
-  backwards-compatible, with two exceptions:
+  backwards-compatible, with a few exceptions:
 
   - Unescaped double quotes are now allowed inside replacement
     fields of f-strings.
+  - Unrecognized backslash escapes in string and byte literals are
+    now syntax errors. (They've been `deprecated in Python since 3.6
+    <https://docs.python.org/3.6/reference/lexical_analysis.html#index-23>`_.)
+  - ``u`` is no longer allowed as a string prefix. (It had no effect,
+    anyway.)
   - A bare `#` is no longer a legal symbol.
 
 * The mangling rules have been refined to account for Python's
@@ -51,6 +56,8 @@ Bug Fixes
 * Tab completion in the Hy REPL now properly unmangles symbol names.
 * `!=` with model objects is now consistent with `=`.
 * Module names supplied to `hy -m` are now mangled.
+* Literal newlines (of all three styles) are now recognized properly
+  in string and bytes literals.
 
 New Features
 ------------------------------
diff --git a/docs/language/syntax.rst b/docs/language/syntax.rst
index 452ec6b7..dd56d539 100644
--- a/docs/language/syntax.rst
+++ b/docs/language/syntax.rst
@@ -60,15 +60,17 @@ following the opening delimiter with no effect on the content. Any leading
 newlines past the first are preserved.
 
 Plain string literals support :ref:`a variety of backslash escapes
-<py:strings>`. To create a "raw string" that interprets all backslashes
-literally, prefix the string with ``r``, as in ``r"slash\not"``. Bracket
-strings are always raw strings and don't allow the ``r`` prefix.
+<py:strings>`. Unrecognized escape sequences are a syntax error. To create
+a "raw string" that interprets all backslashes literally, prefix the string
+with ``r``, as in ``r"slash\not"``. Bracket strings are always raw strings
+and don't allow the ``r`` prefix.
 
 Like Python, Hy treats all string literals as sequences of Unicode characters
 by default. You may prefix a plain string literal (but not a bracket string)
 with ``b`` to treat it as a sequence of bytes.
 
-Unlike Python, Hy only recognizes string prefixes (``r``, etc.) in lowercase.
+Unlike Python, Hy only recognizes string prefixes (``r``, etc.) in lowercase,
+and doesn't allow the no-op prefix ``u``.
 
 .. _syntax-fstrings:
 
diff --git a/hy/lex/hy_reader.py b/hy/lex/hy_reader.py
index 65179ee4..6f84be20 100644
--- a/hy/lex/hy_reader.py
+++ b/hy/lex/hy_reader.py
@@ -250,6 +250,14 @@ class HyReader(Reader):
 
     @reader_for('"')
     def prefixed_string(self, _, prefix=""):
+        prefix_chars = set(prefix)
+        if (
+            len(prefix_chars) != len(prefix)
+            or prefix_chars - set("bfr")
+            or set("bf") <= prefix_chars
+        ):
+            raise LexException.from_reader(f"invalid string prefix {prefix!r}", self)
+
         escaping = False
 
         def quote_closing(c):
@@ -259,6 +267,15 @@ class HyReader(Reader):
                 return 0
             if c == '"' and not escaping:
                 return 1
+            if (
+                escaping
+                and "r" not in prefix
+                and
+                # https://docs.python.org/3/reference/lexical_analysis.html#string-and-bytes-literals
+                c
+                not in ("\n\r\\'\"abfnrtv01234567x" + ("" if "b" in prefix else "NuU"))
+            ):
+                raise LexException.from_reader("invalid escape sequence \\" + c, self)
             escaping = False
             return 0
 
@@ -396,8 +413,10 @@ class HyReader(Reader):
         delim = "".join(delim)
         is_fstring = delim == "f" or delim.startswith("f-")
 
-        # discard single initial newline, if any
-        self.peek_and_getc("\n")
+        # discard single initial newline, if any, accounting for all
+        # three styles of newline
+        self.peek_and_getc("\x0d")
+        self.peek_and_getc("\x0a")
 
         index = -1
 
@@ -447,7 +466,8 @@ class HyReader(Reader):
                     # remove "{" from end of string component
                     s.pop()
                     break
-        res = "".join(s)
+        res = "".join(s).replace("\x0d\x0a", "\x0a").replace("\x0d", "\x0a")
+
         if prefix is not None:
             res = eval(f'{prefix}"""{res}"""')
         if is_fstring:


Test patch:
diff --git a/tests/compilers/test_ast.py b/tests/compilers/test_ast.py
index 8bab8098..0aed4f16 100644
--- a/tests/compilers/test_ast.py
+++ b/tests/compilers/test_ast.py
@@ -499,11 +499,12 @@ Only one leading newline should be removed.
 ]X]''') == '\n\nOnly one leading newline should be removed.\n')
 
 
-@pytest.mark.xfail
 def test_literal_newlines():
     # https://github.com/hylang/hy/issues/2239
     assert s('"\r\nhello\r\nworld"') == "\nhello\nworld"
     assert s('r"\r\nhello\r\nworld"') == "\nhello\nworld"
+    assert s('b"\r\nhello\r\nworld"') == b"\nhello\nworld"
+    assert s('br"\r\nhello\r\nworld"') == b"\nhello\nworld"
     assert s("#[[\r\nhello\r\nworld]]") == "hello\nworld"
     assert s("#[[\rhello\rworld]]") == "hello\nworld"
 
diff --git a/tests/test_lex.py b/tests/test_lex.py
index 615b53bb..0a9957b2 100644
--- a/tests/test_lex.py
+++ b/tests/test_lex.py
@@ -8,10 +8,12 @@ from hy.errors import hy_exc_handler
 from hy.lex import read_many
 from hy.lex.exceptions import LexException, PrematureEndOfInput
 from hy.models import (
+    Bytes,
     Complex,
     Dict,
     Expression,
     Float,
+    FString,
     Integer,
     Keyword,
     List,
@@ -437,13 +439,39 @@ def test_nospace():
     assert entry.end_column == 13
 
 
-def test_escapes():
-    """Ensure we can escape things"""
-    entry = tokenize(r"""(foo "foo\n")""")[0]
-    assert entry[1] == String("foo\n")
+def test_string_prefixes():
+    s = lambda x: tokenize(x)[0]
+
+    assert s(r'b"hello"') == Bytes(b"hello")
+    assert s(r'rb"hello"') == Bytes(b"hello")
+    assert s(r'fr"hello"') == FString([String("hello")])
+
+    for bad in list("zRBFu") + ["bf", "rr", "rbr"]:
+        with lexe():
+            s(bad + '"hello"')
+
 
-    entry = tokenize(r"""(foo r"foo\s")""")[0]
-    assert entry[1] == String(r"foo\s")
+def test_escapes():
+    s = lambda x: tokenize(x)[0]
+
+    # A valid escape sequence
+    assert s(r'"foo\x5a"') == String("fooZ")
+    assert s(r'b"foo\x5a"') == Bytes(b"fooZ")
+    # In a raw string
+    assert s(r'r"foo\x5a"') == String("foo\\x5a")
+    assert s(r'rb"foo\x5a"') == Bytes(b"foo\\x5a")
+    # An invalid escape sequence
+    with lexe():
+        s(r'"foo\s"')
+    with lexe():
+        s(r'b"foo\s"')
+    # In a raw string
+    assert s(r'r"foo\s"') == String("foo\\s")
+    assert s(r'rb"foo\s"') == Bytes(b"foo\\s")
+    # An escape sequence that's valid in strings, but not bytes.
+    assert s(r'"foo\u005a"') == String("fooZ")
+    with lexe():
+        s(r'b"foo\u005a"')
 
 
 def test_unicode_escapes():