Repository: scalameta/scalameta
Pull request #4060 (2024-12-07 02:15:40)

ScalametaParser: refactor parsing modifiers
Helps with #4047.


Patch:
diff --git a/scalameta/parsers/shared/src/main/scala/scala/meta/internal/parsers/ScalametaParser.scala b/scalameta/parsers/shared/src/main/scala/scala/meta/internal/parsers/ScalametaParser.scala
index dfea7a0d4..0a56c145a 100644
--- a/scalameta/parsers/shared/src/main/scala/scala/meta/internal/parsers/ScalametaParser.scala
+++ b/scalameta/parsers/shared/src/main/scala/scala/meta/internal/parsers/ScalametaParser.scala
@@ -41,6 +41,7 @@ class ScalametaParser(input: Input)(implicit dialect: Dialect) {
   private object ReturnTypeContext extends NestedContext
   private object TypeBracketsContext extends NestedContext
   private object PatternTypeContext extends NestedContext
+  private object ExtensionSigContext extends NestedContext
 
   /* ------------- PARSER ENTRY POINTS -------------------------------------------- */
 
@@ -2840,33 +2841,32 @@ class ScalametaParser(input: Input)(implicit dialect: Dialect) {
     }
   }
 
-  private def modifier(isLocal: Boolean): Mod = {
-    val mod = currToken match {
-      case t: Unquote => unquote[Mod](t)
-      case t: Ellipsis => ellipsis[Mod](t, 1)
-      case _: KwAbstract => atCurPosNext(Mod.Abstract())
-      case _: KwFinal => atCurPosNext(Mod.Final())
-      case _: KwSealed => atCurPosNext(Mod.Sealed())
-      case _: KwImplicit => atCurPosNext(Mod.Implicit())
-      case _: KwLazy => atCurPosNext(Mod.Lazy())
-      case _: KwOverride if !isLocal => atCurPosNext(Mod.Override())
-      case _: KwPrivate if !isLocal => privateModifier()
-      case _: KwProtected if !isLocal => protectedModifier()
-      case _ => currToken.text match {
-          case soft.KwInline() => atCurPosNext(Mod.Inline())
-          case soft.KwInfix() => atCurPosNext(Mod.Infix())
-          case soft.KwOpen() if !isLocal => atCurPosNext(Mod.Open())
-          case soft.KwOpaque() => atCurPosNext(Mod.Opaque())
-          case soft.KwTransparent() => atCurPosNext(Mod.Transparent())
-          case soft.KwErased() => atCurPosNext(Mod.Erased())
-          case soft.KwTracked() => atCurPosNext(Mod.Tracked())
-          case n =>
-            val local = if (isLocal) "local " else ""
-            syntaxError(s"${local}modifier expected but $n found", at = currToken)
-        }
-    }
-    newLinesOpt()
-    mod
+  private def badModifier(tok: Token, isLocal: Boolean): Nothing = {
+    val local = if (isLocal) "local " else ""
+    syntaxError(s"${local}modifier expected but ${tok.text} found", at = tok)
+  }
+
+  private def modifier(tok: ModifierKeyword, isLocal: Boolean): Mod = tok match {
+    case _: KwAbstract => atCurPosNext(Mod.Abstract())
+    case _: KwFinal => atCurPosNext(Mod.Final())
+    case _: KwSealed => atCurPosNext(Mod.Sealed())
+    case _: KwImplicit => atCurPosNext(Mod.Implicit())
+    case _: KwLazy => atCurPosNext(Mod.Lazy())
+    case _: KwOverride if !isLocal => atCurPosNext(Mod.Override())
+    case _: KwPrivate if !isLocal => privateModifier()
+    case _: KwProtected if !isLocal => protectedModifier()
+    case _ => badModifier(tok, isLocal)
+  }
+
+  private def modifier(tok: Ident, isLocal: Boolean): Mod = tok.text match {
+    case soft.KwInline() => atCurPosNext(Mod.Inline())
+    case soft.KwInfix() => atCurPosNext(Mod.Infix())
+    case soft.KwOpen() if !isLocal => atCurPosNext(Mod.Open())
+    case soft.KwOpaque() => atCurPosNext(Mod.Opaque())
+    case soft.KwTransparent() => atCurPosNext(Mod.Transparent())
+    case soft.KwErased() => atCurPosNext(Mod.Erased())
+    case soft.KwTracked() => atCurPosNext(Mod.Tracked())
+    case _ => badModifier(tok, isLocal)
   }
 
   def quasiquoteModifier(): Mod = entrypointModifier()
@@ -2930,10 +2930,7 @@ class ScalametaParser(input: Input)(implicit dialect: Dialect) {
       isLocal: Boolean = false,
       isParams: Boolean = false
   ): Unit = {
-    def appendMod: Unit = {
-      val mod = modifier(isLocal)
-      buf += mod
-    }
+    def append(mod: Mod): Unit = { buf += mod; newLinesOpt() }
     // the only things that can come after $mod or $mods are either keywords or names; the former is easy,
     // but in the case of the latter, we need to take care to not hastily parse those names as modifiers
     def continueLoop = peekToken match {
@@ -2943,15 +2940,15 @@ class ScalametaParser(input: Input)(implicit dialect: Dialect) {
     }
     @tailrec
     def loop: Unit = currToken match {
-      case _: Unquote if continueLoop =>
       case _: AtEOL if !isLocal => next(); loop
-      case _: Unquote | _: Ellipsis => appendMod; loop
+      case t: Unquote => if (!continueLoop) { append(unquote[Mod](t)); loop }
+      case t: Ellipsis => append(ellipsis[Mod](t, 1)); loop
       case _: KwCase if peek[KwObject, KwClass] => buf += atCurPosNext(Mod.Case())
-      case _: ModifierKeyword => appendMod; loop
+      case t: ModifierKeyword => append(modifier(t, isLocal)); loop
       case t: Ident if {
             if (isParams) !peek[Colon] && ParamsModifier.matches(t.value)
             else isSoftModifier(currIndex)
-          } => appendMod; loop
+          } => append(modifier(t, isLocal)); loop
       case _ =>
     }
     loop
@@ -2993,36 +2990,65 @@ class ScalametaParser(input: Input)(implicit dialect: Dialect) {
     case _ =>
   }
 
+  private def memberParamClauseGroupOnParen(
+      ownerIsType: Boolean,
+      ownerIsCase: Boolean = false
+  ): Member.ParamClauseGroup = {
+    val tparams = emptyTypeParams
+    autoPos(
+      termParamClausesOnParen(ownerIsType, ownerIsCase = ownerIsCase, ellipsisMaxRank = 3) match {
+        case (x: Quasi) :: Nil if x.rank == 2 => reellipsis[Member.ParamClauseGroup](x, 1)
+        case x => Member.ParamClauseGroup(tparams, x)
+      }
+    )
+  }
+
+  private def memberParamClauseGroupOnBracket(
+      ownerIsType: Boolean,
+      ownerIsCase: Boolean = false,
+      allowUnderscore: Boolean = true
+  ): Member.ParamClauseGroup = autoPos {
+    val tparamClause = typeParamClauseOnBracket(ownerIsType, allowUnderscore = allowUnderscore)
+    val paramClauses = termParamClauses(ownerIsType, ownerIsCase = ownerIsCase)
+    Member.ParamClauseGroup(tparamClause, paramClauses)
+  }
+
+  def memberParamClauseGroup(
+      isFirst: Boolean,
+      ownerIsType: Boolean,
+      ownerIsCase: Boolean = false,
+      allowUnderscore: Boolean = true
+  ): Option[Member.ParamClauseGroup] = {
+    def onFirstParen = Some(memberParamClauseGroupOnParen(ownerIsType, ownerIsCase = ownerIsCase))
+    def onBracket = Some(memberParamClauseGroupOnBracket(
+      ownerIsType,
+      ownerIsCase = ownerIsCase && isFirst,
+      allowUnderscore = allowUnderscore
+    ))
+    currToken match {
+      case _: LeftParen if isFirst => onFirstParen
+      case _: LeftBracket => onBracket
+      case _: EOL => peekToken match {
+          case _: LeftParen if isFirst => next(); onFirstParen
+          case _: LeftBracket => next(); onBracket
+          case _ => None
+        }
+      case _ => None
+    }
+  }
+
   def memberParamClauseGroups(ownerIsType: Boolean): List[Member.ParamClauseGroup] =
-    listBy[Member.ParamClauseGroup] { groups =>
-      @tailrec
-      def iter(): Unit = getAfterOptNewLine(currToken match {
-        case _: LeftParen =>
-          val tparams = emptyTypeParams
-          Some(autoPos {
-            termParamClausesOnParen(ownerIsType, ellipsisMaxRank = 3) match {
-              case (x: Quasi) :: Nil if x.rank == 2 => reellipsis[Member.ParamClauseGroup](x, 1)
-              case x => Member.ParamClauseGroup(tparams, x)
-            }
-          })
-        case _: LeftBracket => Some(autoPos {
-            val tparamClause = typeParamClauseOnBracket(ownerIsType)
-            val paramClauses = termParamClauses(ownerIsType)
-            Member.ParamClauseGroup(tparamClause, paramClauses)
-          })
-        case _ => None
-      }) match {
-        case Some(group) =>
-          groups += group
+    listBy[Member.ParamClauseGroup] { buf =>
+      while ({
+        val pcgOpt = memberParamClauseGroup(isFirst = buf.isEmpty, ownerIsType = ownerIsType)
+        pcgOpt.exists { pcg =>
+          buf += pcg
           // can't have consecutive type clauses (so params must be present)
           // also, only the very last param may contain implicit
-          val ok = group.is[Quasi] || group.paramClauses.lastOption.exists { x =>
-            x.is[Quasi] || !x.mod.is[Mod.Implicit]
-          }
-          if (ok) iter()
-        case _ =>
-      }
-      iter()
+          pcg.is[Quasi] ||
+          pcg.paramClauses.lastOption.exists(pc => pc.is[Quasi] || !pc.mod.is[Mod.Implicit])
+        }
+      }) {}
     }
 
   def termParamClauses(ownerIsType: Boolean, ownerIsCase: Boolean = false): List[Term.ParamClause] =
@@ -3031,60 +3057,76 @@ class ScalametaParser(input: Input)(implicit dialect: Dialect) {
   private def termParamClausesOnParen(
       ownerIsType: Boolean,
       ownerIsCase: Boolean = false,
+      first: Option[Term.ParamClause] = None,
       ellipsisMaxRank: Int = 2
-  ): List[Term.ParamClause] = {
-    var hadModImplicit = false
-    def paramClause(first: Boolean) = autoPos(
-      inParensOnOpenOr {
-        def reduceParams(params: List[Term.Param], mod: Option[Mod.ParamsType] = None) = params
-          .reduceWith { x =>
-            onlyLastParameterCanBeRepeated(x)
-            toParamClause(mod)(x)
-          }
-        def parseParams(mod: Option[Mod.ParamsType] = None) = reduceParams(
-          commaSeparatedWithIndex(termParam(ownerIsCase && first, ownerIsType, mod = mod)),
-          mod
-        )
-        currToken match {
-          case t @ Ellipsis(rank) if rank >= 2 && rank <= ellipsisMaxRank =>
-            reduceParams(List(ellipsis[Term.Param](t)))
-          case _: KwImplicit =>
-            hadModImplicit = true
-            parseParams(Some(atCurPosNext(Mod.Implicit())))
-          case soft.KwUsing() => parseParams(Some(atCurPosNext(Mod.Using())))
-          case _ => parseParams()
-        }
-      }(Term.ParamClause(Nil))
-    )
+  ): List[Term.ParamClause] = listBy[Term.ParamClause] { paramss =>
+    first.foreach(paramss += _)
+    while ({
+      val clause = termParamClauseOnParen(
+        ownerIsType = ownerIsType,
+        ownerIsCase = ownerIsCase && paramss.isEmpty,
+        ellipsisMaxRank = ellipsisMaxRank
+      )
+      paramss += clause
+      val hasModImplicit = clause match {
+        case _: Quasi => false
+        case x => x.mod.exists(_.is[Mod.Implicit])
+      }
+      !hasModImplicit && isAfterOptNewLine[LeftParen]
+    }) {}
+  }
 
-    listBy[Term.ParamClause] { paramss =>
-      paramss += paramClause(true)
-      while (isAfterOptNewLine[LeftParen] && !hadModImplicit) paramss += paramClause(false)
+  private def termParamClauseOnParen(
+      ownerIsType: Boolean,
+      ownerIsCase: Boolean = false,
+      ellipsisMaxRank: Int = 2
+  ): Term.ParamClause = autoPos {
+    def reduceParams(params: List[Term.Param], mod: Option[Mod.ParamsType] = None) = params
+      .reduceWith { x =>
+        onlyLastParameterCanBeRepeated(x)
+        toParamClause(mod)(x)
+      }
+    def parseParams(mod: Option[Mod.ParamsType] = None, ownerIsTypeOverride: => Boolean = false) = {
+      val params = commaSeparatedWithIndex(termParam(
+        ownerIsCase = ownerIsCase,
+        ownerIsType = ownerIsType || ownerIsTypeOverride,
+        mod = mod
+      ))
+      reduceParams(params, mod)
     }
+    inParensOnOpenOr(currToken match {
+      case t @ Ellipsis(rank) if rank >= 2 && rank <= ellipsisMaxRank =>
+        reduceParams(List(ellipsis[Term.Param](t)))
+      case _: KwImplicit => parseParams(Some(atCurPosNext(Mod.Implicit())))
+      case t: Ident if !peek[Colon] =>
+        t.text match {
+          case soft.KwUsing() => parseParams(
+              mod = Some(atCurPosNext(Mod.Using())),
+              ownerIsTypeOverride = ExtensionSigContext.isInside()
+            )
+          case _ => parseParams()
+        }
+      case _ => parseParams()
+    })(Term.ParamClause(Nil))
   }
 
   def termParam(ownerIsCase: Boolean, ownerIsType: Boolean, mod: Option[Mod.ParamsType] = None)(
       paramIdx: Int
   ): Term.Param = autoPos {
     val mods = new ListBuffer[Mod]
+    def appendMod(mod: Mod) = { mods += atCurPosNext(mod); true }
     annotsBuf(mods, skipNewLines = false)
     val numAnnots = mods.length
     if (ownerIsType) modifiersBuf(mods, isParams = true)
-    else {
-      @tailrec
-      def otherMods(): Unit = if (peekToken.isNot[Colon]) {
-        val mod = currToken.text match {
-          case soft.KwInline() => Mod.Inline()
-          case soft.KwErased() => Mod.Erased()
-          case _ => null
-        }
-        if (mod ne null) {
-          mods += atCurPosNext(mod)
-          otherMods()
-        }
-      }
-      otherMods()
-    }
+    else while (currToken match {
+        case t: Ident if !peek[Colon] =>
+          t.text match {
+            case soft.KwInline() => appendMod(Mod.Inline())
+            case soft.KwErased() => appendMod(Mod.Erased())
+            case _ => false
+          }
+        case _ => false
+      }) {}
     val hasExplicitMods = mods.view.drop(numAnnots).exists {
       case _: Mod.Quasi | _: Mod.Erased => false
       case m: Mod.Private => m.within.is[Name.Anonymous]
@@ -3415,12 +3457,12 @@ class ScalametaParser(input: Input)(implicit dialect: Dialect) {
           case t: Ident => termName(t)
           case _ => anonNameEmpty()
         }
-        val tparams =
-          typeParamClauseOpt(ownerIsType = false, allowUnderscore = dialect.allowTypeParamUnderscore)
-        val uparamss =
-          if (at[LeftParen] && soft.KwUsing(peekToken)) termParamClausesOnParen(ownerIsType = false)
-          else Nil
-        if (acceptOpt[Colon]) Some((name, getParamClauseGroup(tparams, uparamss))) else None
+        val pcg = memberParamClauseGroup(
+          isFirst = true,
+          ownerIsType = false,
+          allowUnderscore = dialect.allowTypeParamUnderscore
+        )
+        if (acceptOpt[Colon]) Some((name, pcg)) else None
       }.getOrElse {
 
         /**
@@ -3436,31 +3478,27 @@ class ScalametaParser(input: Input)(implicit dialect: Dialect) {
 
     val decltype = refinement(None).getOrElse(startModType())
 
-    def parents() = {
-      val parents = ListBuffer[Init](
+    def getDefnGiven() = {
+      val inits = templateParentsWithFirst(allowComma = false, allowWithBody = true)(
         autoEndPos(decltype)(initRest(decltype, allowArgss = true, allowTypeSingleton = false))
       )
-      while (at[KwWith] && tryAhead[Ident]) parents += init()
-      parents.toList
-    }
-
-    if (acceptOpt[Equals]) Defn.GivenAlias(mods, sigName, paramClauseGroup, decltype, expr())
-    else if (currToken.isAny[KwWith, LeftParen]) {
-      val inits = parents()
       val body =
         if (acceptOpt[KwWith])
           if (isAfterOptNewLine[LeftBrace]) templateBodyOnLeftBrace(OwnedByGiven)
           else if (at[Indentation.Indent]) autoPos(templateBodyOnIndentRaw(OwnedByGiven))
           else syntaxError("expected '{' or indentation", at = currToken)
-        else {
-          if (inits.lengthCompare(1) > 0) syntaxError("expected 'with' <body>", at = prevToken)
-          emptyTemplateBody()
-        }
+        else if (inits.lengthCompare(1) > 0) syntaxError("expected 'with' <body>", at = prevToken)
+        else emptyTemplateBody()
       val rhs = autoEndPos(decltype)(Template(None, inits, body, Nil))
       Defn.Given(mods, sigName, paramClauseGroup, rhs)
-    } else sigName match {
-      case name: Term.Name => Decl.Given(mods, name, paramClauseGroup, decltype)
-      case _ => syntaxError("abstract givens cannot be anonymous", at = sigName.pos)
+    }
+    currToken match {
+      case _: Equals => next(); Defn.GivenAlias(mods, sigName, paramClauseGroup, decltype, expr())
+      case _: KwWith | _: LeftParen => getDefnGiven()
+      case _ => sigName match {
+          case n: Term.Name => Decl.Given(mods, n, paramClauseGroup, decltype)
+          case n => syntaxError("abstract givens cannot be anonymous", at = n)
+        }
     }
   }
 
@@ -3473,31 +3511,12 @@ class ScalametaParser(input: Input)(implicit dialect: Dialect) {
   def extensionGroupDecl(mods: List[Mod]): Defn.ExtensionGroup = autoEndPos(mods) {
     next() // 'extension'
 
-    val tparams =
-      typeParamClauseOpt(ownerIsType = false, allowUnderscore = dialect.allowTypeParamUnderscore)
-
-    newLineOptWhenFollowedBy[LeftParen]
-
-    val paramss = ListBuffer[Term.ParamClause]()
-
-    def collectUparams(): Unit =
-      while (isAfterOptNewLine[LeftParen] && nextIf(soft.KwUsing(peekToken))) paramss += autoPrevPos {
-        val mod = Some(atCurPos(Mod.Using()))
-        inParensOnOpen(commaSeparatedWithIndex {
-          termParam(ownerIsCase = false, ownerIsType = true, mod = mod)
-        }).reduceWith(toParamClause(mod))
-      }
-
-    collectUparams()
-
-    paramss += autoPos(
-      inParens(List(currToken match {
-        case t @ Ellipsis(2) => ellipsis[Term.Param](t)
-        case _ => termParam(ownerIsCase = false, ownerIsType = false)(0)
-      })).reduceWith(toParamClause(None))
-    )
+    val pcg = ExtensionSigContext.within(memberParamClauseGroup(
+      isFirst = true,
+      ownerIsType = false,
+      allowUnderscore = dialect.allowTypeParamUnderscore
+    ))
 
-    collectUparams()
     newLinesOpt()
 
     def getStats() = statSeq(templateStat())
@@ -3510,7 +3529,7 @@ class ScalametaParser(input: Input)(implicit dialect: Dialect) {
       case _ if isDefIntro(currIndex) => nonLocalDefOrDcl()
       case _ => syntaxError("Extension without extension method", currToken)
     }
-    Defn.ExtensionGroup(getParamClauseGroup(tparams, paramss.toList), body)
+    Defn.ExtensionGroup(pcg, body)
   }
 
   def funDefRest(mods: List[Mod]): Stat = autoEndPos(mods) {
@@ -3525,14 +3544,9 @@ class ScalametaParser(input: Input)(implicit dialect: Dialect) {
       autoPos(Type.Name("Unit"))
     }
 
-    def nonInterleavedParamClauses = {
-      val tparams = typeParamClauseOpt(ownerIsType = false)
-      val paramss = termParamClauses(ownerIsType = false)
-      getParamClauseGroup(tparams, paramss)
-    }
     val paramClauses: List[Member.ParamClauseGroup] =
       if (dialect.allowParamClauseInterleaving) memberParamClauseGroups(ownerIsType = false)
-      else nonInterleavedParamClauses.toList
+      else memberParamClauseGroup(isFirst = true, ownerIsType = false).toList
 
     def defn(declType: Option[Type]) = Defn.Def(mods, name, paramClauses, declType, expr())
 
@@ -3874,12 +3888,21 @@ class ScalametaParser(input: Input)(implicit dialect: Dialect) {
     case _ => initInsideTemplate()
   }
 
-  def templateParents(afterExtend: Boolean = false): List[Init] = {
-    val isSeparator: Token => Boolean =
-      if (afterExtend && dialect.allowCommaSeparatedExtend) _.isAny[KwWith, Comma] else _.is[KwWith]
-    listBy[Init](x => doWhile(x += init())(nextIf(isSeparator(currToken))))
+  def templateParentsWithFirst(allowComma: Boolean, allowWithBody: Boolean = false)(
+      first: Init
+  ): List[Init] = {
+    def impl(isSeparator: => Boolean) =
+      if (!isSeparator) None
+      else Some(listBy[Init] { x => x += first; doWhile { next(); x += init() }(isSeparator) })
+    // whitespace includes Indent and AtEOL
+    (if (allowWithBody) impl(at[KwWith] && !peek[Whitespace, LeftBrace]) else impl(at[KwWith]))
+      .orElse(if (allowComma && dialect.allowCommaSeparatedExtend) impl(at[Comma]) else None)
+      .getOrElse(first :: Nil)
   }
 
+  def templateParents(afterExtend: Boolean = false): List[Init] =
+    templateParentsWithFirst(allowComma = afterExtend)(init())
+
   def derivesClasses(): List[Type] =
     if (atOrPeekAfterEOL(soft.KwDerives(_))) {
       next()


Test patch:
diff --git a/tests/shared/src/test/scala/scala/meta/tests/parsers/dotty/MinorDottySuite.scala b/tests/shared/src/test/scala/scala/meta/tests/parsers/dotty/MinorDottySuite.scala
index cac0a30ca..912642bb8 100644
--- a/tests/shared/src/test/scala/scala/meta/tests/parsers/dotty/MinorDottySuite.scala
+++ b/tests/shared/src/test/scala/scala/meta/tests/parsers/dotty/MinorDottySuite.scala
@@ -323,8 +323,10 @@ class MinorDottySuite extends BaseDottySuite {
     runTestError[Stat]("enum X[T]{ case A[_] extends X[Int] }", "identifier expected")
     runTestError[Stat]("extension [_](x: Int) def inc: Int = x + 1", "identifier expected")
     runTestError[Stat](
-      "given [_](using Ord[T]): Ord[List[T]]{}",
-      "`identifier` expected but `[` found"
+      "given [_](using Ord[T]): Ord[List[T]] with {}",
+      """|<input>:1: error: `identifier` expected but `[` found
+         |given [_](using Ord[T]): Ord[List[T]] with {}
+         |      ^""".stripMargin
     )
   }
 
diff --git a/tests/shared/src/test/scala/scala/meta/tests/parsers/dotty/SignificantIndentationSuite.scala b/tests/shared/src/test/scala/scala/meta/tests/parsers/dotty/SignificantIndentationSuite.scala
index 7167d011a..6ad6b45d2 100644
--- a/tests/shared/src/test/scala/scala/meta/tests/parsers/dotty/SignificantIndentationSuite.scala
+++ b/tests/shared/src/test/scala/scala/meta/tests/parsers/dotty/SignificantIndentationSuite.scala
@@ -666,7 +666,9 @@ class SignificantIndentationSuite extends BaseDottySuite {
          |def fa: Int = 1
          |def fb: Int = 2
          |""".stripMargin,
-      "expected '{' or indentation"
+      """|<input>:2: error: `identifier` expected but `def` found
+         |def fa: Int = 1
+         |^""".stripMargin
     )
 
     runTestError[Stat](