Repository: hylang/hy

Issue:
Oddities in the parsing of quote constructs
`'` isn't allowed in identifiers, but `` ` ``, `~`, and `@` are. Thus, compare:

- `a'b`  → `HySymbol('a'), HyExpression([HySymbol('quote'), HySymbol('b')])`
- ``a`b`` → ``HySymbol('a`b')``
- ``a~b`` → `HySymbol('a~b')`
- ``a~@b`` → `HySymbol('a~@b')`

So, an identifier can have `` ` ``, `~`, or `~@` in the middle, but not at the beginning. And you can't unquote a variable named `@foo` with a tilde, because `~@foo` is parsed as `(unquote-splice foo)`.

Probably we should remove `` ` `` and `~` from the list of identifier characters, but it's less clear what to do with `@`, since ideally that will remain usable as the Python operator of the same name.


Discussion:
Hmm so I guess this is sort of a lose-lose scenario, but as a slightly different thought, what if we:

- Allow ' inside identifiers (e.g. `x'`).
- Disallow @ at the start of multi-character identifiers.

This would be a pretty consistent rule: characters used in quoting aren't allowed at the beginning of multi-character identifiers.

We still end up special-casing for matmul, I almost feel like the cleanest solution would be to change that or ~@ but neither of those options are really *viable* solutions. (Can we even quote the @ operator as it currently stands?)
> Can we even quote the @ operator as it currently stands?

Yep:

```
=> `@
HySymbol('@')
=> '@
HySymbol('@')
=> `~@
<function hyx_Xcommercial_atX at 0x7f9966d55ae8>
```

> We still end up special-casing for matmul, I almost feel like the cleanest solution would be to change that or `~@` but neither of those options are really *viable* solutions.

Why not? `@` is rarely used, so I don't imagine there would be a lot of resistance to renaming it `matmul` or something, and the character used for `~@` seems arbitrary, although it's established in Lisp.
IMO Changing `~@` seems like it would be a rather large breaking change for pretty small benefit (though I guess it could partly be done just via sed).

I *could* go for switching `@` to matmul or similar as a compromise, though we still won't be able to have `'` in identifiers (that's probably just the Haskell-er in me talking).

Resolving patch:
diff --git a/hy/core/result_macros.py b/hy/core/result_macros.py
--- a/hy/core/result_macros.py
+++ b/hy/core/result_macros.py
@@ -223,9 +223,9 @@ def render_quoted_form(compiler, form, level):
 # ------------------------------------------------
 
 
-@pattern_macro(["not", "~"], [FORM], shadow=True)
+@pattern_macro(["not", "bnot"], [FORM], shadow=True)
 def compile_unary_operator(compiler, expr, root, arg):
-    ops = {"not": ast.Not, "~": ast.Invert}
+    ops = {"not": ast.Not, "bnot": ast.Invert}
     operand = compiler.compile(arg)
     return operand + asty.UnaryOp(expr, op=ops[root](), operand=operand.force_expr)
 
diff --git a/hy/reader/hy_reader.py b/hy/reader/hy_reader.py
--- a/hy/reader/hy_reader.py
+++ b/hy/reader/hy_reader.py
@@ -97,7 +97,7 @@ class HyReader(Reader):
     # Components necessary for Reader implementation
     ###
 
-    NON_IDENT = set("()[]{};\"'")
+    NON_IDENT = set("()[]{};\"'`~")
 
     def fill_pos(self, model, start):
         """Attach line/col information to a model.
@@ -283,26 +283,10 @@ def quote_closing(c):
     @reader_for("'", ("quote",))
     @reader_for("`", ("quasiquote",))
     def tag_as(root):
-        def _tag_as(self, _):
-            nc = self.peekc()
-            if (
-                not nc
-                or isnormalizedspace(nc)
-                or self.reader_table.get(nc) == self.INVALID
-            ):
-                raise LexException.from_reader(
-                    "Could not identify the next token.", self
-                )
-            model = self.parse_one_form()
-            return mkexpr(root, model)
-
-        return _tag_as
+        return lambda self, _: mkexpr(root, self.parse_one_form())
 
     @reader_for("~")
     def unquote(self, key):
-        nc = self.peekc()
-        if not nc or isnormalizedspace(nc) or self.reader_table.get(nc) == self.INVALID:
-            return sym(key)
         return mkexpr(
             "unquote" + ("-splice" if self.peek_and_getc("@") else ""),
             self.parse_one_form(),


Test patch:
diff --git a/tests/compilers/test_ast.py b/tests/compilers/test_ast.py
--- a/tests/compilers/test_ast.py
+++ b/tests/compilers/test_ast.py
@@ -81,21 +81,6 @@ def test_ast_valid_if():
     can_compile("(if foo bar baz)")
 
 
-def test_ast_valid_unary_op():
-    "Make sure AST can compile valid unary operator"
-    can_compile("(not 2)")
-    can_compile("(~ 1)")
-
-
-def test_ast_invalid_unary_op():
-    "Make sure AST can't compile invalid unary operator"
-    cant_compile("(not 2 3 4)")
-    cant_compile("(not)")
-    cant_compile("(not 2 3 4)")
-    cant_compile("(~ 2 2 3 4)")
-    cant_compile("(~)")
-
-
 def test_ast_bad_while():
     "Make sure AST can't compile invalid while"
     cant_compile("(while)")
diff --git a/tests/native_tests/operators.hy b/tests/native_tests/operators.hy
--- a/tests/native_tests/operators.hy
+++ b/tests/native_tests/operators.hy
@@ -158,7 +158,7 @@
     ; so we don't allow `^` with 3 arguments, either.
 
 
-(op-and-shadow-test ~
+(op-and-shadow-test bnot
   (forbid (f))
   (assert (= (& (f 0b00101111) 0xFF)
                    0b11010000))
diff --git a/tests/resources/hy_repr_str_tests.txt b/tests/resources/hy_repr_str_tests.txt
--- a/tests/resources/hy_repr_str_tests.txt
+++ b/tests/resources/hy_repr_str_tests.txt
@@ -91,6 +91,7 @@ b"\"double \" quotes\""
 '(f a b)
 '(f #* args #** kwargs)
 [1 [2 3] #(4 #('mysymbol :mykeyword)) {"a" b"hello"} '(f #* a #** b)]
+'(quote)
 
 ;; * Quasiquoting
 
@@ -98,6 +99,10 @@ b"\"double \" quotes\""
 '[1 `[~foo ~@bar] 4]
 '[1 `[~(+ 1 2) ~@(+ [1] [2])] 4]
 '[1 `[~(do (print x 'y) 1)] 4]
+'(quasiquote 1 2 3)
+'[1 `[2 (unquote foo bar)]]
+'[a ~@b]
+'[a ~ @b]
 
 ;; * F-strings
 
diff --git a/tests/test_reader.py b/tests/test_reader.py
--- a/tests/test_reader.py
+++ b/tests/test_reader.py
@@ -86,7 +86,7 @@ def test_lex_single_quote_err():
             '  File "<string>", line 1',
             "    '",
             "    ^",
-            "hy.reader.exceptions.LexException: Could not identify the next token.",
+            "hy.reader.exceptions.PrematureEndOfInput: Premature end of input while attempting to parse one form",
         ],
     )
 
@@ -97,6 +97,26 @@ def test_lex_expression_symbols():
     assert objs == [Expression([Symbol("foo"), Symbol("bar")])]
 
 
+def test_symbol_and_sugar():
+    # https://github.com/hylang/hy/issues/1798
+
+    s = Symbol
+
+    def e(*x):
+        return Expression(x)
+
+    for char, head in (
+        ("'", "quote"),
+        ("`", "quasiquote"),
+        ("~", "unquote"),
+        ("~@", "unquote-splice"),
+    ):
+        for string in (f"a{s1}{char}{s2}b" for s1 in ("", " ") for s2 in ("", " ")):
+            assert tokenize(string) == [s("a"), e(s(head), s("b"))]
+
+    assert tokenize("a~ @b") == tokenize("a ~ @b") == [s("a"), e(s("unquote"), s("@b"))]
+
+
 def test_lex_expression_strings():
     """Test that expressions can produce strings"""
     objs = tokenize('(foo "bar")')