Repository: scalameta/scalameta (scala)

Issue:
Don't emit LF after Outdent, causes wrong token positions
Currently, when ScannerTokens.nextToken works with `prevPos` pointing to last token, and `currPos` referring to a whitespace/comment token ("trivial"), it considers range between `prevPos` and `nextPos` (next non-nontrivial token).

If it determines that an outdent needs to be produced here, the position of the outdent refers to the last newline, in order to include all possible comments.

However, it also produces an LF token after that, and that token uses a position of the **first** newline in the range instead (see #3780), leading to presumably empty trees covering non-empty token ranges.

Pull request description:
Don't allow emitting LF after Outdent, so no more extra LF tokens which might have an earlier position than the Outdent itself leading to empty trees such as Lit.Unit occasionally covering tokens between the LF and the Outdent and breaking logic. Fixes #3856.

Patch:
diff --git a/scalameta/parsers/shared/src/main/scala/scala/meta/internal/parsers/ScannerTokens.scala b/scalameta/parsers/shared/src/main/scala/scala/meta/internal/parsers/ScannerTokens.scala
index 58415de808..5d358a1de3 100644
--- a/scalameta/parsers/shared/src/main/scala/scala/meta/internal/parsers/ScannerTokens.scala
+++ b/scalameta/parsers/shared/src/main/scala/scala/meta/internal/parsers/ScannerTokens.scala
@@ -664,7 +664,7 @@ final class ScannerTokens(val tokens: Tokens)(implicit dialect: Dialect) {
           case _ => None
         }
         val ok = eolPos >= 0 && mightStartStat(next, closeDelimOK = true) &&
-          (prevToken.is[Indentation.Outdent] || canEndStat(prev) || isEndMarker())
+          (canEndStat(prev) || isEndMarker())
         if (ok) strip(regions).map(rs => Right(lastWhitespaceToken(rs, lineIndent))) else None
       }
 


Test patch:
diff --git a/tests/shared/src/test/scala-2.13/scala/meta/tests/parsers/dotty/Scala3PositionSuite.scala b/tests/shared/src/test/scala-2.13/scala/meta/tests/parsers/dotty/Scala3PositionSuite.scala
index 9a765c85cd..e35ab14ba4 100644
--- a/tests/shared/src/test/scala-2.13/scala/meta/tests/parsers/dotty/Scala3PositionSuite.scala
+++ b/tests/shared/src/test/scala-2.13/scala/meta/tests/parsers/dotty/Scala3PositionSuite.scala
@@ -1444,12 +1444,100 @@ class Scala3PositionSuite extends BasePositionSuite(dialects.Scala3) {
        |    // c1</stats0>
        |<thenp>Term.Block aaa
        |    // c1</thenp>
-       |<elsep>Lit.Unit     // c1@@</elsep>
+       |<elsep>Lit.Unit   @@if aa then</elsep>
        |<stats1>Term.If if aa then
        |    aaa</stats1>
        |<elsep>Lit.Unit     aaa@@</elsep>
        |<elsep>Lit.Unit @@</elsep>
        |""".stripMargin,
+    """|BOF [0..0)
+       |KwIf [0..2)
+       |Ident(a) [3..4)
+       |KwThen [5..9)
+       |Indentation.Indent [9..9)
+       |KwIf [12..14)
+       |Ident(aa) [15..17)
+       |KwThen [18..22)
+       |Indentation.Indent [22..22)
+       |Ident(aaa) [27..30)
+       |Indentation.Outdent [40..40)
+       |KwIf [43..45)
+       |Ident(aa) [46..48)
+       |KwThen [49..53)
+       |Indentation.Indent [53..53)
+       |Ident(aaa) [58..61)
+       |Indentation.Outdent [61..61)
+       |Indentation.Outdent [61..61)
+       |EOF [62..62)
+       |""".stripMargin,
+    showFieldName = true
+  )
+
+  checkPositions[Term](
+    """|if (a)
+       |  b.c()
+       |    // c1
+       |    // c2
+       |""".stripMargin,
+    """|<thenp>Term.Block b.c()
+       |    // c1
+       |    // c2</thenp>
+       |<stats0>Term.Apply b.c()</stats0>
+       |<fun>Term.Select b.c</fun>
+       |<argClause>Term.ArgClause ()</argClause>
+       |<elsep>Lit.Unit @@</elsep>
+       |""".stripMargin,
+    showFieldName = true
+  )
+
+  checkPositions[Stat](
+    """|def foo =
+       |  if (a)
+       |    b.c()
+       |      // c1
+       |      // c2
+       |  d.e
+       |""".stripMargin,
+    """|<body>Term.Block if (a)
+       |    b.c()
+       |      // c1
+       |      // c2
+       |  d.e</body>
+       |<stats0>Term.If if (a)
+       |    b.c()
+       |      // c1
+       |      // c2</stats0>
+       |<thenp>Term.Block b.c()
+       |      // c1
+       |      // c2</thenp>
+       |<stats0>Term.Apply b.c()</stats0>
+       |<fun>Term.Select b.c</fun>
+       |<argClause>Term.ArgClause ()</argClause>
+       |<elsep>Lit.Unit   @@d.e</elsep>
+       |<stats1>Term.Select d.e</stats1>
+       |""".stripMargin,
+    """|BOF [0..0)
+       |KwDef [0..3)
+       |Ident(foo) [4..7)
+       |Equals [8..9)
+       |Indentation.Indent [9..9)
+       |KwIf [12..14)
+       |LeftParen [15..16)
+       |Ident(a) [16..17)
+       |RightParen [17..18)
+       |Indentation.Indent [18..18)
+       |Ident(b) [23..24)
+       |Dot [24..25)
+       |Ident(c) [25..26)
+       |LeftParen [26..27)
+       |RightParen [27..28)
+       |Indentation.Outdent [52..52)
+       |Ident(d) [55..56)
+       |Dot [56..57)
+       |Ident(e) [57..58)
+       |Indentation.Outdent [58..58)
+       |EOF [59..59)
+       |""".stripMargin,
     showFieldName = true
   )