Repository: scalameta/scalameta (scala)

Issue:
Unicode escape code in comment crashes scalafmt 3.7.2
## Configuration (required) ##

Please paste the contents of your `.scalafmt.conf` file here:

```
version = "3.7.1"
runner.dialect = "scala3"

align.preset = most
maxColumn = 120
```

## Command-line parameters (required) ##

scalafmt -c .scalafmt.conf example.scala

## Steps

Given code like this:

```scala
// Note: '\u000A' = '\n'
val hello = "hello\u000Aworld!"

@main def main(): Unit =
  println(hello)
```

## Problem

Scalafmt formats code like this:
```scala
org.scalafmt.dynamic.exceptions.PositionExceptionImpl: /home/fabian/[redacted]/example.scala:1: error: unclosed character literal      ] 0 source files formatted
// Note: '\u000A' = '\n'
                ^: /home/fabian/Development/[redacted]/example.scala
```

## Expectation

No crash

## Workaround

~Removing char (`'`) markers around `\u000A`~

Removing the escaped unicode char

Pull request description:
Specifically, do not interpret unicode escapes, double quotes etc. Fixes #3095.

Patch:
diff --git a/scalameta/tokenizers/shared/src/main/scala/scala/meta/internal/tokenizers/CharArrayReader.scala b/scalameta/tokenizers/shared/src/main/scala/scala/meta/internal/tokenizers/CharArrayReader.scala
index e262f61bf7..bd7dd93791 100644
--- a/scalameta/tokenizers/shared/src/main/scala/scala/meta/internal/tokenizers/CharArrayReader.scala
+++ b/scalameta/tokenizers/shared/src/main/scala/scala/meta/internal/tokenizers/CharArrayReader.scala
@@ -55,6 +55,16 @@ class CharArrayReader(input: Input, dialect: Dialect, reporter: Reporter)
     }
   }
 
+  final def nextCommentChar(): Unit = {
+    if (charOffset >= buf.length) {
+      ch = SU
+    } else {
+      ch = buf(charOffset)
+      charOffset += 1
+      checkLineEnd()
+    }
+  }
+
   /**
    * Advance one character, leaving CR;LF pairs intact. This is for use in multi-line strings, so
    * there are no "potential line ends" here.
@@ -132,13 +142,18 @@ class CharArrayReader(input: Input, dialect: Dialect, reporter: Reporter)
 
   /** Handle line ends */
   private def potentialLineEnd(): Unit = {
-    if (ch == LF || ch == FF) {
-      if (!dialect.allowMultilinePrograms) {
-        readerError("line breaks are not allowed in single-line quasiquotes", at = charOffset - 1)
-      }
+    if (checkLineEnd() && !dialect.allowMultilinePrograms) {
+      readerError("line breaks are not allowed in single-line quasiquotes", at = charOffset - 1)
+    }
+  }
+
+  private def checkLineEnd(): Boolean = {
+    val ok = ch == LF || ch == FF
+    if (ok) {
       lastLineStartOffset = lineStartOffset
       lineStartOffset = charOffset
     }
+    ok
   }
 
   /** A new reader that takes off at the current character position */
diff --git a/scalameta/tokenizers/shared/src/main/scala/scala/meta/internal/tokenizers/LegacyScanner.scala b/scalameta/tokenizers/shared/src/main/scala/scala/meta/internal/tokenizers/LegacyScanner.scala
index 303bcedef2..01d416236a 100644
--- a/scalameta/tokenizers/shared/src/main/scala/scala/meta/internal/tokenizers/LegacyScanner.scala
+++ b/scalameta/tokenizers/shared/src/main/scala/scala/meta/internal/tokenizers/LegacyScanner.scala
@@ -23,13 +23,13 @@ class LegacyScanner(input: Input, dialect: Dialect) {
 
   private def isDigit(c: Char) = java.lang.Character isDigit c
   private var openComments = 0
-  protected def putCommentChar(): Unit = nextChar()
+  protected def putCommentChar(): Unit = nextCommentChar()
 
   @tailrec private def skipLineComment(): Unit = ch match {
     case SU | CR | LF =>
     case '$' if isUnquoteNextNoDollar() =>
       syntaxError("can't unquote into single-line comments", at = charOffset - 1)
-    case _ => nextChar(); skipLineComment()
+    case _ => putCommentChar(); skipLineComment()
   }
   private def maybeOpen(): Unit = {
     putCommentChar()
@@ -41,9 +41,10 @@ class LegacyScanner(input: Input, dialect: Dialect) {
   private def maybeClose(): Boolean = {
     putCommentChar()
     (ch == '/') && {
-      putCommentChar()
       openComments -= 1
-      openComments == 0
+      val close = openComments == 0
+      if (close) nextChar() else putCommentChar()
+      close
     }
   }
   @tailrec final def skipNestedComments(): Unit = ch match {
@@ -58,12 +59,12 @@ class LegacyScanner(input: Input, dialect: Dialect) {
   def skipBlockComment(): Unit = skipNestedComments()
 
   private def skipToCommentEnd(isLineComment: Boolean): Unit = {
-    nextChar()
+    putCommentChar()
     if (isLineComment) skipLineComment()
     else {
       openComments = 1
-      val isDocComment = (ch == '*') && { nextChar(); true }
-      if (isDocComment) {
+      if (ch == '*') {
+        putCommentChar()
         // Check for the amazing corner case of /**/
         if (ch == '/')
           nextChar()


Test patch:
diff --git a/tests/shared/src/test/scala/scala/meta/tests/tokenizers/TokenizerSuite.scala b/tests/shared/src/test/scala/scala/meta/tests/tokenizers/TokenizerSuite.scala
index 32ad4e101c..60917190d8 100644
--- a/tests/shared/src/test/scala/scala/meta/tests/tokenizers/TokenizerSuite.scala
+++ b/tests/shared/src/test/scala/scala/meta/tests/tokenizers/TokenizerSuite.scala
@@ -654,6 +654,22 @@ class TokenizerSuite extends BaseTokenizerSuite {
     """.trim.stripMargin)
   }
 
+  test("showRaw with comments - skip unicode escape 1") {
+    val comment = "// Note: '\\u000A' = '\\n'"
+    assertNoDiff(
+      tokenize(comment).map(_.structure).mkString("\n"),
+      Seq("BOF [0..0)", s"$comment [0..24)", "EOF [24..24)").mkString("\n")
+    )
+  }
+
+  test("showRaw with comments - skip unicode escape 2") {
+    val comment = "/* Note: '\\u000A' = '\\n' */"
+    assertNoDiff(
+      tokenize(comment).map(_.structure).mkString("\n"),
+      Seq("BOF [0..0)", s"$comment [0..27)", "EOF [27..27)").mkString("\n")
+    )
+  }
+
   test("interpolation start & end - episode 01") {
     assert(tokenize("q\"\"").map(_.structure).mkString("\n") == """
       |BOF [0..0)