Repository: scalameta/scalameta (scala)

Issue:
"invalid unicode surrogate pair"
Reproduction:

```scala
//> using dep org.scalameta:scalameta_2.13:4.8.14

import scala.meta.*

val file = """
object Hello {
  final val MIN_HIGH_SURROGATE: Char = '\uD800'
}
"""

@main def hello = 
  println(file.parse[Source].get)
```

Whereas the file compiles fine under 2.13 and 3: https://scastie.scala-lang.org/QX3y9ZX6TiyNEqjdFImGyQ

```
Exception in thread "main" <input>:3: error: invalid unicode surrogate pair
  final val MIN_HIGH_SURROGATE: Char = '\uD800'
                                        ^
        at scala.meta.internal.tokenizers.Reporter.readerError(Reporter.scala:20)
        at scala.meta.internal.tokenizers.Reporter.readerError$(Reporter.scala:20)
        at scala.meta.internal.tokenizers.Reporter$$anon$1.readerError(Reporter.scala:33)
        at scala.meta.internal.tokenizers.Reporter.readerError(Reporter.scala:22)
        at scala.meta.internal.tokenizers.Reporter.readerError$(Reporter.scala:22)
        at scala.meta.internal.tokenizers.Reporter$$anon$1.readerError(Reporter.scala:33)
        at scala.meta.internal.tokenizers.CharArrayReader.nextRawChar(CharArrayReader.scala:70)
        at scala.meta.internal.tokenizers.LegacyScanner.fetchSingleQuote$1(LegacyScanner.scala:379)
        at scala.meta.internal.tokenizers.LegacyScanner.fetchToken(LegacyScanner.scala:412)
        at scala.meta.internal.tokenizers.LegacyScanner.nextToken(LegacyScanner.scala:211)
        at scala.meta.internal.tokenizers.LegacyScanner.foreach(LegacyScanner.scala:1011)
        at scala.meta.internal.tokenizers.ScalametaTokenizer.uncachedTokenize(ScalametaTokenizer.scala:24)
        at scala.meta.internal.tokenizers.ScalametaTokenizer.$anonfun$tokenize$1(ScalametaTokenizer.scala:17)
        at scala.collection.concurrent.TrieMap.getOrElseUpdate(TrieMap.scala:962)
        at scala.meta.internal.tokenizers.ScalametaTokenizer.tokenize(ScalametaTokenizer.scala:17)
        at scala.meta.internal.tokenizers.ScalametaTokenizer$$anon$2.apply(ScalametaTokenizer.scala:332)
        at scala.meta.tokenizers.Api$XtensionTokenizeDialectInput.tokenize(Api.scala:25)
        at scala.meta.tokenizers.Api$XtensionTokenizeInputLike.tokenize(Api.scala:14)
        at scala.meta.internal.parsers.ScannerTokens$.apply(ScannerTokens.scala:914)
        at scala.meta.internal.parsers.ScalametaParser.<init>(ScalametaParser.scala:33)
        at scala.meta.parsers.Parse$$anon$1.apply(Parse.scala:35)
        at scala.meta.parsers.Api$XtensionParseDialectInput.parse(Api.scala:25)
        at scala.meta.parsers.Api$XtensionParseInputLike.parse(Api.scala:14)
```

Originally I spotted this issue when Metals was loading Scala Native's standard javalib and crashing there: https://github.com/scala-native/scala-native/blob/main/javalib/src/main/scala/java/lang/Character.scala#L215

And now because of a globally enabled semanticdb plugin I think I have issues with SN's build itself..

Pull request:
Fixes #3402.

Patch:
diff --git a/scalameta/tokenizers/shared/src/main/scala/scala/meta/internal/tokenizers/CharArrayReader.scala b/scalameta/tokenizers/shared/src/main/scala/scala/meta/internal/tokenizers/CharArrayReader.scala
index 20c5579645..e47d37e51b 100644
--- a/scalameta/tokenizers/shared/src/main/scala/scala/meta/internal/tokenizers/CharArrayReader.scala
+++ b/scalameta/tokenizers/shared/src/main/scala/scala/meta/internal/tokenizers/CharArrayReader.scala
@@ -61,12 +61,13 @@ private[meta] case class CharArrayReader private (
     } else {
       begCharOffset = endCharOffset
       val (hi, hiEnd) = readUnicodeChar(buf, endCharOffset)
-      if (!Character.isHighSurrogate(hi)) {
+      val isHiSurrogate = Character.isHighSurrogate(hi)
+      if (isHiSurrogate && hiEnd >= buf.length)
+        readerError("invalid unicode surrogate pair", at = begCharOffset)
+      else if (!isHiSurrogate || ch == '\'' && buf(hiEnd) == '\'' && !wasMultiChar) {
         ch = hi
         endCharOffset = hiEnd
-      } else if (hiEnd >= buf.length)
-        readerError("invalid unicode surrogate pair", at = begCharOffset)
-      else {
+      } else {
         val (lo, loEnd) = readUnicodeChar(buf, hiEnd)
         if (!Character.isLowSurrogate(lo))
           readerError("invalid unicode surrogate pair", at = begCharOffset)


Test patch:
diff --git a/tests/shared/src/test/scala/scala/meta/tests/tokenizers/TokenizerSuite.scala b/tests/shared/src/test/scala/scala/meta/tests/tokenizers/TokenizerSuite.scala
index 5cc6d659b6..7dd3edd7ef 100644
--- a/tests/shared/src/test/scala/scala/meta/tests/tokenizers/TokenizerSuite.scala
+++ b/tests/shared/src/test/scala/scala/meta/tests/tokenizers/TokenizerSuite.scala
@@ -1218,4 +1218,10 @@ class TokenizerSuite extends BaseTokenizerSuite {
     )
   }
 
+  test("#3402") {
+    val code = "val MIN_HIGH_SURROGATE = '\\uD800'"
+    val res = dialects.Scala212(code).tokenize
+    assertEquals(res.get.toString, code)
+  }
+
 }