Repository: effekt-lang/effekt (scala)

Issue:
Inconsistency between JS and Scheme backends regarding Unicode (double-)escapes
Hello dear team, 

I seem to have encountered an inconsistency between the behaviour of programs compiled for the JS backend and for the Chez Scheme backend in the context of doubly escaped Unicode codepoints in strings.

The following line, compiled for the Node.js backend and executed in a Windows console:

`println("\\u000A is the Unicode representation of \\n")`

prints, as expected:

> \u000A is the Unicode representation of \n

However, when compiled for the Chez Scheme backend, it prints:

> \012 is the Unicode representation of \n

The generated Scheme code is `(println_1 "\\012 is the Unicode representation of \\n")`.

Apparently, the Scheme version partly honours the double escape (insofar as it preserves it, and doesn't print an actual newline), but nevertheless converts the hexadecimal Unicode value into its octal representation. 
Triple-quoting doesn't prevent that either.

Unless I'm missing or misunderstanding something, this currently makes it necessary to use a workaround like 

`println("\\" ++ "u000A")`

to print 

> \u000A

literally on the Scheme backend.

(Tested on Effekt from 2024-09-13, as of commit 19089d3)

Kind regards,
Michael

Pull request description:
This gets us half-way there for escapes in Strings:
- Allow escapes in character literals, i.e. `'\n'` is a newline character
    - Note: This disallows `'\'` for \, but requires `'\\'`
    - Fixes `'\n' == '\t'`  (cmp. #521)
- Check that character literals consist of only one codepoint
- Report wrong escape sequences in parser, but generate the error in lexer - this only happens for SOME of those
    - We should generate error tokens everywhere instead, different issue
- Escape ourselves in Lexer and build up a string
- Add additional escapes: `\$` (because of unquotes), `\u{...}`
- In chez, escape all characters in one function, use chez's unicode handling functions for characters beyond 3 octals
   - Fixes #596

Patch:
diff --git a/effekt/shared/src/main/scala/effekt/Lexer.scala b/effekt/shared/src/main/scala/effekt/Lexer.scala
index a7e07409c..32049d7c1 100644
--- a/effekt/shared/src/main/scala/effekt/Lexer.scala
+++ b/effekt/shared/src/main/scala/effekt/Lexer.scala
@@ -305,7 +305,7 @@ class Lexer(source: Source) {
               }
             // Last `}` is not be considered as part of string interpolation. Let the parser fail if braces don't match
             // and just continue lexing
-            case `{{` | `"` | `"""` => nextToken()
+            case `{{` | `"` | `"""` | `'` => nextToken()
           }
         } else nextToken()
       kind match {
@@ -366,26 +366,36 @@ class Lexer(source: Source) {
       this match {
         case `"` => 1
         case `"""` => 3
+        case `'` => 1
+      }
+
+    def first: Char =
+      this match {
+        case `"` | `"""` => '"'
+        case `'` => '\''
       }
 
     override def toString: String =
       this match {
         case `"` => "\""
         case `"""` => "\"\"\""
+        case `'` => "'"
       }
 
     def isMultiline: Boolean =
       this match {
         case `"` => false
         case `"""` => true
+        case `'` => false
       }
   }
   case object `"` extends StrDelim
   case object `"""` extends StrDelim
+  /** Delimiter for characters */
+  case object `'` extends StrDelim
 
   /** Matches a string literal -- both single- and multi-line strings.
-   * Strings may contain space characters (e.g. \n, \t, \r), which are stored unescaped for single-line strings and
-   * escaped for multi-line strings.
+   * Strings may contain space characters (e.g. \n, \t, \r), which are stored unescaped.
    * Strings may also contain unquotes, i.e., "f = ${x + 1}" that include arbitrary expressions.
    */
   def matchString(delim: StrDelim, continued: Boolean = false): TokenKind = {
@@ -393,76 +403,97 @@ class Lexer(source: Source) {
     val offset = delim.offset
     val st = if (continued) start else start + offset
     var endString = false
+    val stringContent = StringBuilder()
+
     delimiters.push(delim)
     while (!endString) {
       peek() match {
-        case Some('\"') => {
+        case Some(c) if c == delim.first => {
           consume()
           delim match {
-            case `"""` => {
-              if (peekN(1).contains('\"') && peekN(2).contains('\"')) {
-                consume(); consume()
-                delimiters.pop()
-                endString = true
-              }
+            case `"""`  if (peekN(1).contains('\"') && peekN(2).contains('\"')) => {
+              consume(); consume()
+              delimiters.pop()
+              endString = true
             }
             case `"` => {
               delimiters.pop()
               endString = true
             }
+            case `'` => {
+              delimiters.pop()
+              endString = true
+            }
+            case _ => stringContent.addOne(c)
           }
         }
-        case Some('\n') if delim == `"` =>
+        case Some('\n') if !delim.isMultiline =>
           return err("Linebreaks are not allowed in single-line strings.")
         // handle escape sequences
-        case Some('\\') => {
+        case Some('\\') if !delim.isMultiline => {
           expect('\\')
           peek() match {
-            case Some('\"') | Some('\\') | Some('n') | Some('t') | Some('r') => consume()
+            case Some('"') => consume(); stringContent.addOne('"')
+            case Some('n') => consume(); stringContent.addOne('\n')
+            case Some('t') => consume(); stringContent.addOne('\t')
+            case Some('r') => consume(); stringContent.addOne('\r')
+            case Some('\\') => consume(); stringContent.addOne('\\')
+            case Some('$') => consume(); stringContent.addOne('$')
             case Some('u') => {
               consume()
-              consumeWhile(c => !c.isWhitespace && c != '"')
+
+              def isHexDigit(c: Char) = c.isDigit || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F')
+
+              val hexCodePoint = peek() match {
+                case Some(c) if isHexDigit(c) =>
+                  val escapeStart = current
+                  consumeWhile(isHexDigit)
+                  slice(escapeStart, current)
+                case Some('{') =>
+                  consume()
+                  val escapeStart = current
+                  consumeWhile(c => c != '}')
+                  expect('}')
+                  slice(escapeStart, current - 1)
+                case _ => err("Invalid escape sequence.", current - 1, current)
+              }
+
+              try {
+                val codePoint = Integer.parseInt(hexCodePoint, 16)
+                stringContent.append(String.valueOf(Character.toChars(codePoint)))
+              } catch {
+                case e: NumberFormatException => err(e.toString)
+              }
             }
             case _ => err("Invalid escape sequence.", current - 1, current)
           }
         }
         case Some('$') if peekN(2).contains('{') => {
-          val s = slice(st, current)
-          return TokenKind.Str(s, delim.isMultiline)
+          return TokenKind.Str(stringContent.mkString, delim.isMultiline)
         }
         case None =>
           return err("Unterminated string.", start, start + offset)
-        case _ =>
+        case Some(c) =>
           consume()
+          stringContent.addOne(c)
       }
     }
-    val s = slice(st, current - offset)
-
-    val stringContent =
-      if (delim.isMultiline) {
-        // Do not unescape multiline strings at all for multiline strings (see comments in PR #495)
-        s
-      } else {
-        // Unescape the following sequences: control: `\b`, `\t`, `\n`, `\f`, `\r`; escape:  `\\`, `\"`, `\'`
-        // For now, we only support a subset determined by `StringContext.processEscapes`
-        try {
-          StringContext.processEscapes(s)
-        } catch {
-          case e: StringContext.InvalidEscapeException => err("Contains invalid escape sequence.", e.index, e.index)
-        }
-      }
 
-    TokenKind.Str(stringContent, delim.isMultiline)
+    TokenKind.Str(stringContent.mkString, delim.isMultiline)
   }
 
   /** Matches a character: '.+' */
   def matchChar(): TokenKind = {
-    if (peekMatches(_ == '\'')) err("Empty character literal.")
-    consumeWhile(_ != '\'')
-    expect('\'')
-    // exclude opening and closing '
-    val cs = slice(start + 1, current - 1)
-    TokenKind.Chr(cs.codePointAt(0))
+    matchString(`'`) match {
+      case TokenKind.Str("", _) =>
+        TokenKind.Error(LexerError("Empty character literal", start, current))
+      case TokenKind.Str(cs, _) if (cs.codePointCount(0, cs.length) > 1) =>
+        TokenKind.Error(LexerError("Character literal consists " +
+            "of multiple code points.", start, current))
+      case TokenKind.Str(cs, _) =>
+        TokenKind.Chr(cs.codePointAt(0))
+      case err => err
+    }
   }
 
   lazy val hexadecimal = ('a' to 'f') ++ ('A' to 'F') ++ ('0' to '9')
diff --git a/effekt/shared/src/main/scala/effekt/RecursiveDescent.scala b/effekt/shared/src/main/scala/effekt/RecursiveDescent.scala
index 702a3c031..dfaf30028 100644
--- a/effekt/shared/src/main/scala/effekt/RecursiveDescent.scala
+++ b/effekt/shared/src/main/scala/effekt/RecursiveDescent.scala
@@ -60,7 +60,12 @@ class RecursiveDescent(positions: Positions, tokens: Seq[Token], source: Source)
   // always points to the latest non-space position
   var position: Int = 0
 
-  def peek: Token = tokens(position)
+  extension(token: Token) def failOnErrorToken: Token = token.kind match {
+    case TokenKind.Error(err) => fail(err.msg)
+    case _ => token
+  }
+
+  def peek: Token = tokens(position).failOnErrorToken
 
   /**
    * Negative lookahead
@@ -76,7 +81,7 @@ class RecursiveDescent(positions: Positions, tokens: Seq[Token], source: Source)
     def go(position: Int, offset: Int): Token =
       if position >= tokens.length then fail("Unexpected end of file")
 
-      tokens(position) match {
+      tokens(position).failOnErrorToken match {
         case token if isSpace(token.kind) => go(position + 1, offset)
         case token if offset <= 0 => token
         case _ => go(position + 1, offset - 1)
@@ -94,7 +99,7 @@ class RecursiveDescent(positions: Positions, tokens: Seq[Token], source: Source)
 
   def hasNext(): Boolean = position < tokens.length
   def next(): Token =
-    val t = tokens(position)
+    val t = tokens(position).failOnErrorToken
     skip()
     t
 
diff --git a/effekt/shared/src/main/scala/effekt/generator/chez/Transformer.scala b/effekt/shared/src/main/scala/effekt/generator/chez/Transformer.scala
index 722181059..3fa50aac0 100644
--- a/effekt/shared/src/main/scala/effekt/generator/chez/Transformer.scala
+++ b/effekt/shared/src/main/scala/effekt/generator/chez/Transformer.scala
@@ -12,6 +12,7 @@ import util.messages.{ INTERNAL_ERROR, NOT_SUPPORTED }
 
 import scala.language.implicitConversions
 import scala.util.matching.Regex
+import scala.collection.mutable
 
 object TransformerMonadic extends Transformer {
 
@@ -218,7 +219,7 @@ trait Transformer {
   def toChez(expr: Expr): chez.Expr = expr match {
     case Literal((), _)         => chez.RawValue("#f")
 
-    case Literal(s: String, _)  => ChezString(adaptEscapes(escape(s)))
+    case Literal(s: String, _)  => escape(s)
     case Literal(b: Boolean, _) => if (b) chez.RawValue("#t") else chez.RawValue("#f")
     case l: Literal             => chez.RawValue(l.value.toString)
     case ValueVar(id, _)        => chez.Variable(nameRef(id))
@@ -257,11 +258,26 @@ trait Transformer {
     List(getter, setter)
   }
 
-  def escape(scalaString: String): String =
-    scalaString.foldLeft(StringBuilder()) { (acc, c) =>
-      escapeSeqs.get(c) match {
-        case Some(s) => acc ++= s
-        case None => acc += c
-      }
-    }.toString()
+  def escape(scalaString: String): chez.Expr = {
+    val parts = mutable.ListBuffer[chez.Expr]()
+    val strPart = StringBuilder()
+    scalaString.codePoints().forEach {
+      case c if escapeSeqs.contains(c.toChar) => strPart.append(escapeSeqs(c.toChar))
+      case c if c >= 32 && c <= 126 => strPart.append(String.valueOf(Character.toChars(c)))
+      case c if c < 8 * 8 * 8 =>
+        strPart.append("\\" + Integer.toString(c, 8).reverse.padTo(3, '0').reverse)
+      case c =>
+        parts.addOne(chez.RawValue("\"" ++ strPart.mkString ++ "\""))
+        strPart.clear()
+        parts.addOne(chez.Call(chez.RawExpr("string"), chez.Call(chez.RawExpr("integer->char"),
+          chez.RawExpr("#x" ++ c.toHexString))))
+    }
+    parts.addOne(chez.RawValue("\"" ++ strPart.mkString ++ "\""))
+
+    if (parts.size == 1) {
+      parts(0)
+    } else {
+      chez.Call(chez.RawExpr("string-append"), parts.toList)
+    }
+  }
 }
diff --git a/effekt/shared/src/main/scala/effekt/generator/chez/package.scala b/effekt/shared/src/main/scala/effekt/generator/chez/package.scala
index 98e09b145..b13fb4628 100644
--- a/effekt/shared/src/main/scala/effekt/generator/chez/package.scala
+++ b/effekt/shared/src/main/scala/effekt/generator/chez/package.scala
@@ -73,16 +73,3 @@ def generateConstructor(id: Symbol, fields: List[Symbol]): List[chez.Def] = {
 
   List(record, matcher)
 }
-
-
-private val unicodeRx = Regex("""[\\]u([0-9a-fA-F]{1,4})""")
-
-// unicode escape sequences \u001b need to be translated to \033
-// where 033 is an octal value left-padded with 0 to exactly three digits.
-def hexToOctal(str: String) =
-  val n = Integer.parseInt(str, 16)
-  val octal = Integer.toString(n, 8)
-  octal.reverse.padTo(3, '0').reverse
-
-def adaptEscapes(str: String) =
-  unicodeRx.replaceAllIn(str, m => "\\\\" + s"${hexToOctal(m.group(1))}")
diff --git a/examples/pos/unicode_escapes.check b/examples/pos/unicode_escapes.check
new file mode 100644
index 000000000..ccc689618
--- /dev/null
+++ b/examples/pos/unicode_escapes.check
@@ -0,0 +1,3 @@
+\u000A is the Unicode representation of \n
+💩💩
+false
\ No newline at end of file
diff --git a/examples/pos/unicode_escapes.effekt b/examples/pos/unicode_escapes.effekt
new file mode 100644
index 000000000..5e7e453e6
--- /dev/null
+++ b/examples/pos/unicode_escapes.effekt
@@ -0,0 +1,5 @@
+def main() = {
+  println("\\u000A is the Unicode representation of \\n")
+  println("💩\u1f4a9")
+  println('\n' == '\t')
+}
\ No newline at end of file


Test patch:
diff --git a/effekt/jvm/src/test/scala/effekt/LexerTests.scala b/effekt/jvm/src/test/scala/effekt/LexerTests.scala
index 6bd05c1c6..53f013dae 100644
--- a/effekt/jvm/src/test/scala/effekt/LexerTests.scala
+++ b/effekt/jvm/src/test/scala/effekt/LexerTests.scala
@@ -97,19 +97,20 @@ class LexerTests extends munit.FunSuite {
     assertTokensEq("\" \\t \"", Str(" \t ", false), EOF)
     assertFailure("\"\\k\"")
     assertTokensEq("\"\\u001b\"", Str("\u001b", false), EOF)
+    assertTokensEq("\"\\u{001b}\"", Str("\u001b", false), EOF)
   }
 
   test("characters") {
     assertTokensEq("' '", Chr(' '), EOF)
     assertTokensEq("'😅'", Chr(0x1F605), EOF)
-    assertTokensEq("'\\'", Chr('\\'), EOF)
-    assertTokensEq("'\n'", Chr('\n'), EOF)
-    assertTokensEq("'\t'", Chr('\t'), EOF)
+    assertTokensEq("'\\\\'", Chr('\\'), EOF)
+    assertTokensEq("'\\n'", Chr('\n'), EOF)
+    assertTokensEq("'\\t'", Chr('\t'), EOF)
     assertTokensEq("\\u0000", Chr(0), EOF)
     assertTokensEq("\\uFFFF", Chr(0xFFFF), EOF)
     assertTokensEq("\\u10FFFF{ val", Chr(0x10FFFF), `{`, `val`, EOF)
     assertTokensEq("val s =\\u0000+ 1", `val`, Ident("s"), `=`, Chr(0), `+`, Integer(1), EOF)
-    assertFailure("''")
+    assertTokensEq("''", Error(LexerError("Empty character literal", 0, 2)), EOF)
   }
 
   test("multi line strings") {