Repository: scalameta/scalameta (scala)

Issue:
Scala 3 indentation syntax: False positive "outdent expected" error
### Describe the bug

To reproduce:

1. Write this

```scala
//> using scala 3.5.2

def demo: Option[Unit] =
  for
    _ <- Option(42).map: _ =>
      ???

    _ <- Option(42)
  yield ()

```

2. There's an error at the second `<-`

<img width="635" alt="image" src="https://github.com/user-attachments/assets/d03788ec-5d5e-4443-a2e9-9734649c5c8d">


### Expected behavior

No error

### Operating system

macOS

### Editor/Extension

VS Code

### Version of Metals

v1.4.0

### Extra context or search terms

- works fine in 1.3.5
- compiler is happy with the syntax
- removing the empty line between the generators, or replacing it with a comment, removes the error

Pull request description:
Fixes #4008.

Patch:
diff --git a/scalameta/parsers/shared/src/main/scala/scala/meta/internal/parsers/ScannerTokens.scala b/scalameta/parsers/shared/src/main/scala/scala/meta/internal/parsers/ScannerTokens.scala
index 1e784c6890..98552e62b9 100644
--- a/scalameta/parsers/shared/src/main/scala/scala/meta/internal/parsers/ScannerTokens.scala
+++ b/scalameta/parsers/shared/src/main/scala/scala/meta/internal/parsers/ScannerTokens.scala
@@ -351,35 +351,50 @@ final class ScannerTokens(val tokens: Tokens)(implicit dialect: Dialect) {
         xs: List[SepRegion],
         wasDone: Boolean = false,
         head: TokenRef = null,
-        last: TokenRef = null
+        last: TokenRef = null,
+        multiEOL: Boolean = false
     )(implicit f: List[SepRegion] => OutdentInfo): Either[List[SepRegion], (TokenRef, TokenRef)] =
       if (!wasDone) f(xs) match {
-        case null => mkOutdentsOpt(xs, true, head, last)
+        case null => mkOutdentsOpt(xs, true, head, last, multiEOL)
         case OutdentInfo(outdent, rs, done) =>
           val isDone = done || (rs eq xs)
-          if (outdent eq null) mkOutdentsOpt(rs, isDone, head, last)
+          if (outdent eq null) mkOutdentsOpt(rs, isDone, head, last, multiEOL)
           else {
             val tr = mkOutdentTo(outdent, rs)
             val newHead = if (head eq null) tr else { last.next = tr; head }
-            mkOutdentsOpt(rs, isDone, newHead, tr)
+            mkOutdentsOpt(rs, isDone, newHead, tr, multiEOL)
           }
       }
       else if (head eq null) Left(xs)
       else {
         if (currNonTrivial) last.next = currRef(xs)
+        else if (multiEOL) last.next = eofRefAt(xs, last.pointPos, multiEOL = true)
         Right((head, last))
       }
 
-    def mkOutdentsT[A](xs: List[SepRegion])(lt: List[SepRegion] => A)(
+    def mkOutdentsT[A](xs: List[SepRegion], multiEOL: Boolean = false)(lt: List[SepRegion] => A)(
         rt: (TokenRef, TokenRef) => A
-    )(implicit f: List[SepRegion] => OutdentInfo): A = mkOutdentsOpt(xs).fold(lt, rt.tupled)
+    )(implicit f: List[SepRegion] => OutdentInfo): A = mkOutdentsOpt(xs, multiEOL = multiEOL)
+      .fold(lt, rt.tupled)
 
-    def mkOutdents(xs: List[SepRegion])(implicit f: List[SepRegion] => OutdentInfo): TokenRef =
-      mkOutdentsT(xs)(currRef(_))((x, _) => x)
+    def mkOutdents(xs: List[SepRegion], multiEOL: Boolean = false)(implicit
+        f: List[SepRegion] => OutdentInfo
+    ): TokenRef = mkOutdentsT(xs, multiEOL)(currRef)((x, _) => x)
 
-    def currRef(regions: List[SepRegion], next: TokenRef = null): TokenRef =
+    def currRef(regions: List[SepRegion]): TokenRef = currAndNextRef(regions, null)
+    def currAndNextRef(regions: List[SepRegion], next: TokenRef): TokenRef =
       TokenRef(regions, curr, currPos, next)
 
+    def eolRefFor(rs: List[SepRegion], out: Token, eolPos: Int) =
+      TokenRef(rs, out, eolPos, nextPos, eolPos, null)
+    def eofRefAt(rs: List[SepRegion], eolPos: Int, multiEOL: Boolean): TokenRef = {
+      val token = tokens(eolPos)
+      val out =
+        if (!multiEOL || token.is[MultiEOL]) token
+        else LFLF(token.input, token.dialect, token.start, tokens(indentPos).end)
+      eolRefFor(rs, out, eolPos)
+    }
+
     def outdentThenCurrRef(
         ri: RegionIndent,
         rs: List[SepRegion],
@@ -411,7 +426,7 @@ final class ScannerTokens(val tokens: Tokens)(implicit dialect: Dialect) {
             if dialect.allowSignificantIndentation && curr.pos.endLine == next.pos.startLine =>
           val regions = markRegions()
           val nextRegions = RegionIndent(findIndent(sepRegions)) :: regions
-          currRef(regions, mkIndent(currPos, nextPos, nextRegions))
+          currAndNextRef(regions, mkIndent(currPos, nextPos, nextRegions))
         case _: KwCase | _: LeftBrace => currRef(markRegions())
         case _ => currRef(sepRegions)
       }
@@ -611,8 +626,6 @@ final class ScannerTokens(val tokens: Tokens)(implicit dialect: Dialect) {
       if (isTrailingComma) nextToken(curr, currPos, currPos + 1, sepRegionsOrig)
       else nonTrivial(getNonTrivialRegions(sepRegionsOrig))
     else {
-      def eolRef(rs: List[SepRegion], out: Token, eolPos: Int) =
-        TokenRef(rs, out, eolPos, nextPos, eolPos, null)
       @tailrec
       def findFirstEOL(pos: Int): Int =
         if (pos > indentPos) -1 else if (tokens(pos).is[AtEOL]) pos else findFirstEOL(pos + 1)
@@ -633,11 +646,7 @@ final class ScannerTokens(val tokens: Tokens)(implicit dialect: Dialect) {
       def lastWhitespaceToken(rs: List[SepRegion], lineIndent: Int) = {
         val addRegionLine = lineIndent >= 0 && isIndented(rs, lineIndent)
         val regions = if (addRegionLine) RegionLine(lineIndent) :: rs else rs
-        val token = tokens(eolPos)
-        val out =
-          if (!multiEOL || token.is[MultiEOL]) token
-          else LFLF(token.input, token.dialect, token.start, tokens(indentPos).end)
-        eolRef(regions, out, eolPos)
+        eofRefAt(regions, eolPos, multiEOL)
       }
 
       def stripIfCanProduceLF(regions: List[SepRegion]) = {
@@ -686,7 +695,7 @@ final class ScannerTokens(val tokens: Tokens)(implicit dialect: Dialect) {
         def getInfixLF(invalid: Option[String]) = Some(Right {
           val lf = tokens(indentPos)
           val out = InfixLF(lf.input, lf.dialect, lf.start, lf.end, invalid)
-          eolRef(sepRegionsOrig, out, indentPos)
+          eolRefFor(sepRegionsOrig, out, indentPos)
         })
         isLeadingInfix(regions) match {
           case LeadingInfix.Yes => getInfixLF(None)
@@ -854,7 +863,8 @@ final class ScannerTokens(val tokens: Tokens)(implicit dialect: Dialect) {
 
           val regionsToRes: List[SepRegion] => Option[Either[List[SepRegion], TokenRef]] =
             if (prevToken.is[Indentation]) _ => None
-            else mkOutdentsT(_)(getIndentIfNeeded)((x, _) => Some(Right(x)))(getOutdentInfo)
+            else
+              mkOutdentsT(_, multiEOL)(getIndentIfNeeded)((x, _) => Some(Right(x)))(getOutdentInfo)
 
           def onlyWithoutLF() = sepRegionsOrig match {
             case (ri: SepRegionIndented) :: _ if ri.indent < nextIndent =>


Test patch:
diff --git a/tests/shared/src/test/scala-2.13/scala/meta/tests/parsers/dotty/Scala3PositionSuite.scala b/tests/shared/src/test/scala-2.13/scala/meta/tests/parsers/dotty/Scala3PositionSuite.scala
index 8929023a6a..704c1d9304 100644
--- a/tests/shared/src/test/scala-2.13/scala/meta/tests/parsers/dotty/Scala3PositionSuite.scala
+++ b/tests/shared/src/test/scala-2.13/scala/meta/tests/parsers/dotty/Scala3PositionSuite.scala
@@ -2079,6 +2079,7 @@ class Scala3PositionSuite extends BasePositionSuite(dialects.Scala3) {
        |RightArrow [43..45)
        |Ident(foo) [46..49)
        |Indentation.Outdent [50..50)
+       |LFLF [50..65)
        |KwVal [68..71)
        |Ident(qux) [72..75)
        |Equals [76..77)
@@ -2151,6 +2152,7 @@ class Scala3PositionSuite extends BasePositionSuite(dialects.Scala3) {
        |RightArrow [52..54)
        |Ident(qux) [55..58)
        |Indentation.Outdent [59..59)
+       |LFLF [59..72)
        |KwDef [74..77)
        |Ident(quux) [78..82)
        |Equals [83..84)
@@ -2274,6 +2276,7 @@ class Scala3PositionSuite extends BasePositionSuite(dialects.Scala3) {
        |RightArrow [93..95)
        |Ident(qux) [96..99)
        |Indentation.Outdent [114..114)
+       |LFLF [114..138)
        |RightBrace [140..141)
        |Indentation.Outdent [141..141)
        |EOF [142..142)
@@ -2442,6 +2445,7 @@ class Scala3PositionSuite extends BasePositionSuite(dialects.Scala3) {
        |Ident(qux) [114..117)
        |Indentation.Outdent [135..135)
        |Indentation.Outdent [150..150)
+       |LFLF [150..174)
        |RightBrace [176..177)
        |Indentation.Outdent [177..177)
        |EOF [178..178)
@@ -2619,6 +2623,7 @@ class Scala3PositionSuite extends BasePositionSuite(dialects.Scala3) {
        |Indentation.Outdent [149..149)
        |Indentation.Outdent [162..162)
        |Indentation.Outdent [173..173)
+       |LFLF [173..182)
        |RightBrace [182..183)
        |EOF [184..184)
        |""".stripMargin,
@@ -2793,6 +2798,7 @@ class Scala3PositionSuite extends BasePositionSuite(dialects.Scala3) {
        |Indentation.Outdent [161..161)
        |Indentation.Outdent [172..172)
        |Indentation.Outdent [180..180)
+       |LFLF [180..181)
        |EOF [181..181)
        |""".stripMargin,
     showFieldName = true
@@ -2834,6 +2840,7 @@ class Scala3PositionSuite extends BasePositionSuite(dialects.Scala3) {
        |Ident(*) [53..54)
        |Constant.Int(2) [55..56)
        |Indentation.Outdent [68..68)
+       |LFLF [68..69)
        |EOF [69..69)
        |""".stripMargin,
     showFieldName = true
diff --git a/tests/shared/src/test/scala/scala/meta/tests/parsers/dotty/ControlSyntaxSuite.scala b/tests/shared/src/test/scala/scala/meta/tests/parsers/dotty/ControlSyntaxSuite.scala
index 3b92f51c30..c809535019 100644
--- a/tests/shared/src/test/scala/scala/meta/tests/parsers/dotty/ControlSyntaxSuite.scala
+++ b/tests/shared/src/test/scala/scala/meta/tests/parsers/dotty/ControlSyntaxSuite.scala
@@ -4284,6 +4284,7 @@ class ControlSyntaxSuite extends BaseDottySuite {
                     |LF [54..55)
                     |RightBrace [59..60)
                     |Indentation.Outdent [61..61)
+                    |LFLF [61..62)
                     |Underscore [64..65)
                     |LeftArrow [66..68)
                     |Ident(d) [69..70)
@@ -4349,6 +4350,7 @@ class ControlSyntaxSuite extends BaseDottySuite {
                     |LF [54..55)
                     |RightBrace [59..60)
                     |Indentation.Outdent [61..61)
+                    |LFLF [61..62)
                     |Underscore [64..65)
                     |LeftArrow [66..68)
                     |Ident(d) [69..70)
@@ -4377,4 +4379,66 @@ class ControlSyntaxSuite extends BaseDottySuite {
     runTestAssert[Stat](code, layout)(tree)
   }
 
+  test("#4008") {
+    val code = """|for
+                  |    _ <- Option(42).map: _ =>
+                  |      ???
+                  |
+                  |    _ <- Option(43)
+                  |yield ()
+                  |
+                  |""".stripMargin
+
+    val struct = """|BOF [0..0)
+                    |KwFor [0..3)
+                    |Indentation.Indent [3..3)
+                    |Underscore [8..9)
+                    |LeftArrow [10..12)
+                    |Ident(Option) [13..19)
+                    |LeftParen [19..20)
+                    |Constant.Int(42) [20..22)
+                    |RightParen [22..23)
+                    |Dot [23..24)
+                    |Ident(map) [24..27)
+                    |Colon [27..28)
+                    |Underscore [29..30)
+                    |RightArrow [31..33)
+                    |Indentation.Indent [33..33)
+                    |Ident(???) [40..43)
+                    |Indentation.Outdent [44..44)
+                    |LFLF [44..45)
+                    |Underscore [49..50)
+                    |LeftArrow [51..53)
+                    |Ident(Option) [54..60)
+                    |LeftParen [60..61)
+                    |Constant.Int(43) [61..63)
+                    |RightParen [63..64)
+                    |Indentation.Outdent [64..64)
+                    |KwYield [65..70)
+                    |LeftParen [71..72)
+                    |RightParen [72..73)
+                    |EOF [75..75)
+                    |""".stripMargin.nl2lf
+    assertTokenizedAsStructureLines(code, struct)
+
+    val layout = """|for (_ <- Option(42).map {
+                    |  _ => ???
+                    |}; _ <- Option(43)) yield ()
+                    |""".stripMargin
+    val tree = Term.ForYield(
+      Term.EnumeratorsBlock(List(
+        Enumerator.Generator(
+          Pat.Wildcard(),
+          Term.Apply(
+            Term.Select(Term.Apply(tname("Option"), List(lit(42))), tname("map")),
+            List(blk(Term.Function(List(tparam("_")), tname("???"))))
+          )
+        ),
+        Enumerator.Generator(Pat.Wildcard(), Term.Apply(tname("Option"), List(lit(43))))
+      )),
+      Lit.Unit()
+    )
+    runTestAssert[Stat](code, layout)(tree)
+  }
+
 }