Repository: scalameta/scalameta
Pull request #2636 (2022-02-23 13:42:14)

ScalametaParser: tailrec recursion for batchSource


Patch:
diff --git a/scalameta/parsers/shared/src/main/scala/scala/meta/internal/parsers/LazyTokenIterator.scala b/scalameta/parsers/shared/src/main/scala/scala/meta/internal/parsers/LazyTokenIterator.scala
index 83e6137fa..b7a0d55d4 100644
--- a/scalameta/parsers/shared/src/main/scala/scala/meta/internal/parsers/LazyTokenIterator.scala
+++ b/scalameta/parsers/shared/src/main/scala/scala/meta/internal/parsers/LazyTokenIterator.scala
@@ -161,11 +161,11 @@ private[parsers] class LazyTokenIterator private (
         if (pos < currPos) pos else currPos - 1
       } else {
         scannerTokens(i) match {
-          case _: LF | _: FF =>
+          case _: EOL =>
             iter(i + 1, if (pos == prevPos) i else pos, 0)
-          case _: Space | _: Tab if indent >= 0 =>
+          case _: HSpace if indent >= 0 =>
             iter(i + 1, pos, indent + 1)
-          case Whitespace() =>
+          case _: Whitespace =>
             iter(i + 1, pos, indent)
           case _: Comment if indent < 0 || outdent <= indent =>
             iter(i + 1, i + 1, -1)
@@ -194,7 +194,7 @@ private[parsers] class LazyTokenIterator private (
         if (i == scannerTokens.length) (-1, null)
         else
           scannerTokens(i) match {
-            case Trivia() => iter(i + 1)
+            case _: Trivia => iter(i + 1)
             case t => (i, t)
           }
       iter(currPos + 1)
@@ -471,10 +471,10 @@ private[parsers] class LazyTokenIterator private (
       else {
         val token = scannerTokens(pos)
         token match {
-          case _: LF | _: BOF => (acc, pos)
+          case _: EOL | _: BOF => (acc, pos)
           case AsMultilineComment(c) => (multilineCommentIndent(c), pos)
           case _: Comment => countIndentInternal(pos - 1)
-          case _: Space | _: Tab => countIndentInternal(pos - 1, acc + 1)
+          case _: HSpace => countIndentInternal(pos - 1, acc + 1)
           case _ => (-1, -1)
         }
       }
diff --git a/scalameta/parsers/shared/src/main/scala/scala/meta/internal/parsers/ScalametaParser.scala b/scalameta/parsers/shared/src/main/scala/scala/meta/internal/parsers/ScalametaParser.scala
index 3be208db8..e2e7ae1dd 100644
--- a/scalameta/parsers/shared/src/main/scala/scala/meta/internal/parsers/ScalametaParser.scala
+++ b/scalameta/parsers/shared/src/main/scala/scala/meta/internal/parsers/ScalametaParser.scala
@@ -156,10 +156,11 @@ class ScalametaParser(input: Input)(implicit dialect: Dialect) { parser =>
 
     @tailrec
     def isNextEOL(t: Token): Boolean = {
-      val next = t.nextSafe
-      if (next.is[LineEnd] || next.is[MultilineComment]) true
-      else if ((next.is[Whitespace] && !next.is[LineEnd]) || next.is[Comment]) isNextEOL(next)
-      else false
+      t.nextSafe match {
+        case _: AtEOL | MultilineComment() => true
+        case x: Trivia => isNextEOL(x)
+        case _ => false
+      }
     }
 
     dialect.allowSignificantIndentation && token.is[Colon] && isNextEOL(token)
@@ -488,47 +489,47 @@ class ScalametaParser(input: Input)(implicit dialect: Dialect) { parser =>
 
   /* ---------- TREE CONSTRUCTION ------------------------------------------- */
 
-  def ellipsis[T <: Tree](rank: Int, extraSkip: => Unit = {})(implicit astInfo: AstInfo[T]): T =
+  def ellipsis[T <: Tree: AstInfo](ell: Ellipsis, rank: Int, extraSkip: => Unit = {}): T = {
+    if (ell.rank != rank) {
+      syntaxError(Messages.QuasiquoteRankMismatch(ell.rank, rank), at = ell)
+    }
+    ellipsis(ell, extraSkip)
+  }
+
+  def ellipsis[T <: Tree: AstInfo](ell: Ellipsis): T = ellipsis(ell, {})
+
+  def ellipsis[T <: Tree](ell: Ellipsis, extraSkip: => Unit)(implicit astInfo: AstInfo[T]): T = {
+    if (!dialect.allowUnquotes) {
+      syntaxError(s"$dialect doesn't support ellipses", at = ell)
+    }
     autoPos {
-      token match {
-        case ellipsis: Ellipsis =>
-          if (dialect.allowUnquotes) {
-            if (ellipsis.rank != rank) {
-              syntaxError(Messages.QuasiquoteRankMismatch(ellipsis.rank, rank), at = ellipsis)
-            } else {
-              next()
-              extraSkip
-              val tree = {
-                val result = token match {
-                  case LeftParen() => inParens(unquote[T])
-                  case LeftBrace() => inBraces(unquote[T])
-                  case Unquote() => unquote[T]
-                  case _ => syntaxError(s"$$, ( or { expected but ${token.name} found", at = token)
-                }
-                result match {
-                  case quasi: Quasi =>
-                    // NOTE: In the case of an unquote nested directly under ellipsis, we get a bit of a mixup.
-                    // Unquote's pt may not be directly equal unwrapped ellipsis's pt, but be its refinement instead.
-                    // For example, in `new { ..$stats }`, ellipsis's pt is List[Stat], but quasi's pt is Term.
-                    // This is an artifact of the current implementation, so we just need to keep it mind and work around it.
-                    require(
-                      classTag[T].runtimeClass.isAssignableFrom(quasi.pt) &&
-                        debug(ellipsis, result, result.structure)
-                    )
-                    copyPos(quasi)(astInfo.quasi(quasi.rank, quasi.tree))
-                  case other =>
-                    other
-                }
-              }
-              astInfo.quasi(rank, tree)
-            }
-          } else {
-            syntaxError(s"$dialect doesn't support ellipses", at = ellipsis)
-          }
-        case _ =>
-          unreachable(debug(token))
+      next()
+      extraSkip
+      val tree = {
+        val result = token match {
+          case LeftParen() => inParens(unquote[T])
+          case LeftBrace() => inBraces(unquote[T])
+          case Unquote() => unquote[T]
+          case t => syntaxError(s"$$, ( or { expected but ${t.name} found", at = t)
+        }
+        result match {
+          case quasi: Quasi =>
+            // NOTE: In the case of an unquote nested directly under ellipsis, we get a bit of a mixup.
+            // Unquote's pt may not be directly equal unwrapped ellipsis's pt, but be its refinement instead.
+            // For example, in `new { ..$stats }`, ellipsis's pt is List[Stat], but quasi's pt is Term.
+            // This is an artifact of the current implementation, so we just need to keep it mind and work around it.
+            require(
+              classTag[T].runtimeClass.isAssignableFrom(quasi.pt) &&
+                debug(ell, result, result.structure)
+            )
+            copyPos(quasi)(astInfo.quasi(quasi.rank, quasi.tree))
+          case other =>
+            other
+        }
       }
+      astInfo.quasi(ell.rank, tree)
     }
+  }
 
   def unquote[T <: Tree: AstInfo](advance: Boolean = true): T = autoPos {
     token match {
@@ -577,16 +578,16 @@ class ScalametaParser(input: Input)(implicit dialect: Dialect) { parser =>
   ): List[T] = {
     val ts = new ListBuffer[T]
     @tailrec
-    def iter(sep: Boolean): Unit = {
-      if (token.is[Ellipsis]) {
-        ts += ellipsis[T](1)
+    def iter(sep: Boolean): Unit = token match {
+      case t: Ellipsis =>
+        ts += ellipsis[T](t, 1)
         iter(false)
-      } else if (sep) {
+      case _ if sep =>
         ts += part
         iter(false)
-      } else if (acceptOpt[Sep]) {
+      case _ if acceptOpt[Sep] =>
         iter(true)
-      }
+      case _ =>
     }
     iter(!sepFirst)
     ts.toList
@@ -669,8 +670,8 @@ class ScalametaParser(input: Input)(implicit dialect: Dialect) { parser =>
       }
       @tailrec
       def paramOrType(): Tree = token match {
-        case Ellipsis(rank) =>
-          ellipsis[Tree](rank)
+        case t: Ellipsis =>
+          ellipsis[Tree](t)
         case Unquote() =>
           unquote[Tree]
         case KwImplicit() if !hasImplicits =>
@@ -916,10 +917,10 @@ class ScalametaParser(input: Input)(implicit dialect: Dialect) { parser =>
             Type.Placeholder(typeBounds())
           else
             Type.Name(s"${value}_")
-        case Literal() =>
+        case _: Literal =>
           if (dialect.allowLiteralTypes) literal()
           else syntaxError(s"$dialect doesn't support literal types", at = path())
-        case Ident("-") if dialect.allowLiteralTypes && ahead(token.is[NumericLiteral]) =>
+        case Ident("-") if dialect.allowLiteralTypes && ahead(token.is[NumericConstant[_]]) =>
           next()
           literal(isNegated = true)
         case _ =>
@@ -2046,7 +2047,7 @@ class ScalametaParser(input: Input)(implicit dialect: Dialect) { parser =>
     if (!isUnaryOp) simpleExpr(allowRepeated)
     else {
       val op = termName()
-      if (op.value == "-" && token.is[NumericLiteral])
+      if (op.value == "-" && token.is[NumericConstant[_]])
         simpleExprRest(autoEndPos(op)(literal(isNegated = true)), canApply = true)
       else {
         simpleExpr0(allowRepeated = true) match {
@@ -2073,7 +2074,7 @@ class ScalametaParser(input: Input)(implicit dialect: Dialect) { parser =>
           Success(macroQuotedIdent())
         case MacroSplicedIdent() =>
           Success(macroSplicedIdent())
-        case Literal() =>
+        case _: Literal =>
           Success(literal())
         case Interpolation.Id(_) =>
           Success(interpolateTerm())
@@ -2253,8 +2254,8 @@ class ScalametaParser(input: Input)(implicit dialect: Dialect) { parser =>
       inParens(token match {
         case RightParen() =>
           (Nil, false)
-        case tok: Ellipsis if tok.rank == 2 =>
-          (List(ellipsis[Term](2)), false)
+        case t @ Ellipsis(2) =>
+          (List(ellipsis[Term](t)), false)
         case _ =>
           val using = if (token.is[soft.KwUsing]) {
             next(); true
@@ -2329,8 +2330,8 @@ class ScalametaParser(input: Input)(implicit dialect: Dialect) { parser =>
     val cases = new ListBuffer[Case]
     @tailrec
     def iter(): Unit = token match {
-      case _: Ellipsis =>
-        cases += ellipsis[Case](1, accept[KwCase])
+      case t: Ellipsis =>
+        cases += ellipsis[Case](t, 1, accept[KwCase])
         while (token.is[StatSep]) next()
         iter
       case CaseIntro() =>
@@ -2366,7 +2367,7 @@ class ScalametaParser(input: Input)(implicit dialect: Dialect) { parser =>
 
   def enumerator(isFirst: Boolean, allowNestedIf: Boolean = true): List[Enumerator] = token match {
     case _: KwIf if !isFirst => autoPos(Enumerator.Guard(guard().get)) :: Nil
-    case _: Ellipsis => ellipsis[Enumerator](1) :: Nil
+    case t: Ellipsis => ellipsis[Enumerator](t, 1) :: Nil
     case _: Unquote if ahead(!token.is[Equals] && !token.is[LeftArrow]) =>
       unquote[Enumerator] :: Nil // support for q"for ($enum1; ..$enums; $enum2)"
     case _ => generator(!isFirst, allowNestedIf)
@@ -2617,7 +2618,7 @@ class ScalametaParser(input: Input)(implicit dialect: Dialect) { parser =>
               !isBackquoted && ((value.head.isLower && value.head.isLetter) || value.head == '_' || isCapitalAllowed)
             case _ => false
           }
-          if (token.is[NumericLiteral]) {
+          if (token.is[NumericConstant[_]]) {
             sid match {
               case Term.Name("-") =>
                 return autoEndPos(sid)(literal(isNegated = true))
@@ -2650,7 +2651,7 @@ class ScalametaParser(input: Input)(implicit dialect: Dialect) { parser =>
         case Underscore() =>
           next()
           Pat.Wildcard()
-        case Literal() =>
+        case _: Literal =>
           literal()
         case Interpolation.Id(_) =>
           interpolatePat()
@@ -2751,7 +2752,7 @@ class ScalametaParser(input: Input)(implicit dialect: Dialect) { parser =>
   def modifier(): Mod =
     autoPos(token match {
       case Unquote() => unquote[Mod]
-      case Ellipsis(_) => ellipsis[Mod](1)
+      case t: Ellipsis => ellipsis[Mod](t, 1)
       case KwAbstract() => next(); Mod.Abstract()
       case KwFinal() => next(); Mod.Final()
       case KwSealed() => next(); Mod.Sealed()
@@ -2806,7 +2807,7 @@ class ScalametaParser(input: Input)(implicit dialect: Dialect) { parser =>
 
   def ctorModifiers(): Option[Mod] = token match {
     case Unquote() if ahead(token.is[LeftParen]) => Some(unquote[Mod])
-    case Ellipsis(_) => Some(ellipsis[Mod](1))
+    case t: Ellipsis => Some(ellipsis[Mod](t, 1))
     case KwPrivate() => Some(accessModifier())
     case KwProtected() => Some(accessModifier())
     case _ => None
@@ -2817,7 +2818,7 @@ class ScalametaParser(input: Input)(implicit dialect: Dialect) { parser =>
     else
       token match {
         case Unquote() => Some(unquote[Mod])
-        case Ellipsis(_) => Some(ellipsis[Mod](1))
+        case t: Ellipsis => Some(ellipsis[Mod](t, 1))
         case Ident("+") => Some(autoPos({ next(); Mod.Covariant() }))
         case Ident("-") => Some(autoPos({ next(); Mod.Contravariant() }))
         case _ => None
@@ -2880,8 +2881,8 @@ class ScalametaParser(input: Input)(implicit dialect: Dialect) { parser =>
           if (token.is[Unquote]) annots += unquote[Mod.Annot]
           else annots += autoEndPos(StartPosPrev)(Mod.Annot(initInsideAnnotation(allowArgss)))
           true
-        case _: Ellipsis if ahead(token.is[At]) =>
-          annots += ellipsis[Mod.Annot](1, next())
+        case t: Ellipsis if ahead(token.is[At]) =>
+          annots += ellipsis[Mod.Annot](t, 1, next())
           true
         case _ => false
       }) {
@@ -2899,8 +2900,8 @@ class ScalametaParser(input: Input)(implicit dialect: Dialect) { parser =>
     def paramClause(first: Boolean): List[Term.Param] = token match {
       case RightParen() =>
         Nil
-      case tok: Ellipsis if tok.rank == 2 =>
-        List(ellipsis[Term.Param](2))
+      case t @ Ellipsis(2) =>
+        List(ellipsis[Term.Param](t))
       case _ =>
         var parsedUsing = false
         if (token.is[KwImplicit]) {
@@ -3091,7 +3092,7 @@ class ScalametaParser(input: Input)(implicit dialect: Dialect) { parser =>
               @inline def getBound(): Type = {
                 next()
                 token match {
-                  case t: Ellipsis => ellipsis[Type](1)
+                  case t: Ellipsis => ellipsis[Type](t, 1)
                   case _ => typ()
                 }
               }
@@ -3933,7 +3934,7 @@ class ScalametaParser(input: Input)(implicit dialect: Dialect) { parser =>
 
   def init() = {
     token match {
-      case Ellipsis(_) => ellipsis[Init](1)
+      case t: Ellipsis => ellipsis[Init](t, 1)
       case _ => initInsideTemplate()
     }
   }
@@ -3954,7 +3955,7 @@ class ScalametaParser(input: Input)(implicit dialect: Dialect) { parser =>
       next()
       val deriving = ListBuffer[Type]()
       def readInit() = token match {
-        case Ellipsis(_) => deriving += ellipsis[Type](1)
+        case t: Ellipsis => deriving += ellipsis[Type](t, 1)
         case _ => deriving += startModType()
       }
       readInit()
@@ -4108,7 +4109,7 @@ class ScalametaParser(input: Input)(implicit dialect: Dialect) { parser =>
     case DefIntro() => nonLocalDefOrDcl(secondaryConstructorAllowed = true)
     case EndMarkerIntro() => endMarker()
     case ExprIntro() => stat(expr(location = NoStat, allowRepeated = true))
-    case Ellipsis(_) => ellipsis[Stat](1)
+    case t: Ellipsis => ellipsis[Stat](t, 1)
   }
 
   def quasiquoteStat(): Stat = {
@@ -4156,6 +4157,15 @@ class ScalametaParser(input: Input)(implicit dialect: Dialect) { parser =>
       errorMsg: String = "illegal start of definition"
   ): List[T] = {
     val stats = new ListBuffer[T]
+    statSeqBuf(stats, statpf, errorMsg)
+    stats.toList
+  }
+
+  def statSeqBuf[T <: Tree: AstInfo](
+      stats: ListBuffer[T],
+      statpf: PartialFunction[Token, T],
+      errorMsg: String = "illegal start of definition"
+  ): Unit = {
     val isIndented = acceptOpt[Indentation.Indent]
     val statpfAdd = statpf.runWith(stats += _)
 
@@ -4165,12 +4175,11 @@ class ScalametaParser(input: Input)(implicit dialect: Dialect) { parser =>
       else syntaxError(errorMsg + s" ${token.name}", at = token)
     }
     if (isIndented) accept[Indentation.Outdent]
-    stats.toList
   }
 
-  def topStat: PartialFunction[Token, Stat] = {
-    case Ellipsis(_) =>
-      ellipsis[Stat](1)
+  val topStat: PartialFunction[Token, Stat] = {
+    case t: Ellipsis =>
+      ellipsis[Stat](t, 1)
     case Unquote() =>
       unquote[Stat]
     case KwPackage() =>
@@ -4233,8 +4242,8 @@ class ScalametaParser(input: Input)(implicit dialect: Dialect) { parser =>
         nonLocalDefOrDcl(enumCaseAllowed, secondaryConstructorAllowed)
       case Unquote() =>
         unquote[Stat]
-      case Ellipsis(_) =>
-        ellipsis[Stat](1)
+      case t: Ellipsis =>
+        ellipsis[Stat](t, 1)
       case EndMarkerIntro() =>
         endMarker()
       case ExprIntro() =>
@@ -4253,7 +4262,7 @@ class ScalametaParser(input: Input)(implicit dialect: Dialect) { parser =>
   }
 
   def refineStat(): Option[Stat] = token match {
-    case _: Ellipsis => Some(ellipsis[Stat](1))
+    case t: Ellipsis => Some(ellipsis[Stat](t, 1))
     case DclIntro() =>
       defOrDclOrSecondaryCtor(Nil) match {
         case stat if stat.isRefineStat => Some(stat)
@@ -4309,8 +4318,8 @@ class ScalametaParser(input: Input)(implicit dialect: Dialect) { parser =>
         if (!token.is[CaseDefEnd]) acceptStatSep()
       case StatSep() =>
         next()
-      case _: Ellipsis =>
-        stats += ellipsis[Stat](1)
+      case t: Ellipsis =>
+        stats += ellipsis[Stat](t, 1)
       case EndMarkerIntro() =>
         stats += endMarker()
       case _ =>
@@ -4361,37 +4370,46 @@ class ScalametaParser(input: Input)(implicit dialect: Dialect) { parser =>
         token.isNot[LeftBrace] && !isColonEol(token)
       }
     }
-    def bracelessPackageStats(): List[Stat] = token match {
-      case _: EOF => Nil
+    val buf = new ListBuffer[Stat]
+    @tailrec
+    def bracelessPackageStats(f: List[Stat] => List[Stat]): List[Stat] = token match {
+      case _: EOF => f(buf.toList)
       case StatSep() =>
         next()
-        bracelessPackageStats()
+        bracelessPackageStats(f)
       case _: KwPackage if !ahead(token.is[KwObject]) =>
         val startPos = in.tokenPos
         next()
         val qid = qualId()
         def inPackage(stats: => List[Stat]) = {
-          val pkg = autoEndPos(startPos)(Pkg(qid, stats))
+          buf += autoEndPos(startPos)(Pkg(qid, stats))
           acceptStatSepOpt()
-          pkg +: bracelessPackageStats()
         }
         token match {
-          case _: LeftBrace => inPackage(inBraces(statSeq(statpf)))
+          case _: LeftBrace =>
+            inPackage(inBraces(statSeq(statpf)))
+            bracelessPackageStats(f)
           case t if isColonEol(t) =>
             next()
             in.observeIndented()
             inPackage(indented(statSeq(statpf)))
-          case _ => List(autoEndPos(startPos)(Pkg(qid, bracelessPackageStats())))
+            bracelessPackageStats(f)
+          case _ => bracelessPackageStats(x => f(List(autoEndPos(startPos)(Pkg(qid, x)))))
         }
-      case _: LeftBrace => inBraces(statSeq(statpf))
-      case _ => statSeq(statpf)
+      case _: LeftBrace =>
+        inBraces(statSeqBuf(buf, statpf))
+        f(buf.toList)
+      case _ =>
+        statSeqBuf(buf, statpf)
+        f(buf.toList)
     }
     if (inBracelessPackage()) {
       val startPos = in.tokenPos
       next()
-      Source(List(autoEndPos(startPos)(Pkg(qualId(), bracelessPackageStats()))))
+      Source(List(autoEndPos(startPos)(Pkg(qualId(), bracelessPackageStats(identity)))))
     } else {
-      Source(statSeq(statpf))
+      statSeqBuf(buf, statpf)
+      Source(buf.toList)
     }
   }
 }
diff --git a/scalameta/parsers/shared/src/main/scala/scala/meta/internal/parsers/ScannerTokens.scala b/scalameta/parsers/shared/src/main/scala/scala/meta/internal/parsers/ScannerTokens.scala
index 9b29f519c..7a52870c5 100644
--- a/scalameta/parsers/shared/src/main/scala/scala/meta/internal/parsers/ScannerTokens.scala
+++ b/scalameta/parsers/shared/src/main/scala/scala/meta/internal/parsers/ScannerTokens.scala
@@ -55,8 +55,7 @@ class ScannerTokens(tokens: Tokens, input: Input)(implicit dialect: Dialect) {
 
   @tailrec
   private final def getStrictAfterSafe(token: Token): Token = {
-    if (token.is[Space] || token.is[Tab] || token.is[Comment])
-      getStrictAfterSafe(token.nextSafe)
+    if (token.is[HSpace] || token.is[Comment]) getStrictAfterSafe(token.nextSafe)
     else token
   }
 
@@ -166,7 +165,7 @@ class ScannerTokens(tokens: Tokens, input: Input)(implicit dialect: Dialect) {
         token.is[Ident] &&
         token.text == "end" &&
         token.strictNext.is[EndMarkerWord] &&
-        (token.next.strictNext.is[LineEnd] || token.next.strictNext.is[EOF])
+        (token.next.strictNext.is[AtEOL] || token.next.strictNext.is[EOF])
       }
     }
     // then  else  do  catch  finally  yield  match
@@ -272,11 +271,7 @@ class ScannerTokens(tokens: Tokens, input: Input)(implicit dialect: Dialect) {
     @classifier
     trait Modifier {
       def unapply(token: Token): Boolean = {
-        token.is[KwAbstract] || token.is[KwFinal] ||
-        token.is[KwSealed] || token.is[KwImplicit] ||
-        token.is[KwLazy] || token.is[KwPrivate] ||
-        token.is[KwProtected] || token.is[KwOverride] ||
-        token.is[SoftModifier]
+        token.is[ModifierKeyword] || token.is[SoftModifier]
       }
     }
 
diff --git a/scalameta/parsers/shared/src/main/scala/scala/meta/internal/parsers/TokenClassifiers.scala b/scalameta/parsers/shared/src/main/scala/scala/meta/internal/parsers/TokenClassifiers.scala
index 6106bfc1d..b319a11c5 100644
--- a/scalameta/parsers/shared/src/main/scala/scala/meta/internal/parsers/TokenClassifiers.scala
+++ b/scalameta/parsers/shared/src/main/scala/scala/meta/internal/parsers/TokenClassifiers.scala
@@ -5,64 +5,6 @@ import scala.meta.internal.classifiers.classifier
 import scala.meta.tokens.Token
 import scala.meta.tokens.Token._
 
-@classifier
-trait CloseDelim {
-  def unapply(token: Token): Boolean = {
-    token.is[RightBrace] || token.is[RightBracket] || token.is[RightParen] ||
-    token.is[Indentation.Outdent]
-  }
-}
-
-@classifier
-trait Literal {
-  def unapply(token: Token): Boolean = token match {
-    case Constant.Int(_) => true
-    case Constant.Long(_) => true
-    case Constant.Float(_) => true
-    case Constant.Double(_) => true
-    case Constant.Char(_) => true
-    case Constant.Symbol(_) => true
-    case Constant.String(_) => true
-    case KwTrue() => true
-    case KwFalse() => true
-    case KwNull() => true
-    case _ => false
-  }
-}
-
-@classifier
-trait NumericLiteral {
-  def unapply(token: Token): Boolean = token match {
-    case Constant.Int(_) => true
-    case Constant.Long(_) => true
-    case Constant.Float(_) => true
-    case Constant.Double(_) => true
-    case _ => false
-  }
-}
-
-@classifier
-trait Whitespace {
-  def unapply(token: Token): Boolean = {
-    token.is[Space] || token.is[Tab] ||
-    token.is[LineEnd] || token.is[FF]
-  }
-}
-
-@classifier
-trait LineEnd {
-  def unapply(token: Token): Boolean = {
-    token.is[LF] || token.is[LFLF] || token.is[CR]
-  }
-}
-
-@classifier
-trait Trivia {
-  def unapply(token: Token): Boolean = {
-    token.is[Whitespace] || token.is[Comment]
-  }
-}
-
 @classifier
 trait MultilineComment {
   def unapply(token: Token): Boolean =
diff --git a/scalameta/tokens/shared/src/main/scala/scala/meta/tokens/Token.scala b/scalameta/tokens/shared/src/main/scala/scala/meta/tokens/Token.scala
index 863d7fc0d..8fbcbe224 100644
--- a/scalameta/tokens/shared/src/main/scala/scala/meta/tokens/Token.scala
+++ b/scalameta/tokens/shared/src/main/scala/scala/meta/tokens/Token.scala
@@ -1,6 +1,7 @@
 package scala.meta
 package tokens
 
+import scala.math.ScalaNumber
 import scala.meta.internal.tokens._
 import scala.meta.inputs._
 import scala.meta.classifiers._
@@ -21,91 +22,112 @@ import scala.meta.internal.prettyprinters._
 }
 
 object Token {
+  // Literals (include some keywords from above, constants, interpolations and xml)
+  @branch trait Literal extends Token
+  @branch abstract class Constant[A] extends Literal {
+    val value: A
+  }
+  @branch abstract class NumericConstant[A <: ScalaNumber] extends Constant[A]
+
+  @branch trait Keyword extends Token
+  @branch trait ModifierKeyword extends Keyword
+
+  @branch trait Trivia extends Token
+  @branch trait Whitespace extends Trivia
+  @branch trait HSpace extends Whitespace
+  @branch trait AtEOL extends Whitespace
+  @branch trait EOL extends AtEOL
+
+  @branch trait Symbolic extends Token
+  @branch trait SymbolicKeyword extends Symbolic
+  @branch trait Punct extends Symbolic
+  @branch trait OpenDelim extends Punct
+  @branch trait CloseDelim extends Punct
+
   // Identifiers
   @freeform("identifier") class Ident(value: String) extends Token
 
   // Alphanumeric keywords
-  @fixed("abstract") class KwAbstract extends Token
-  @fixed("case") class KwCase extends Token
-  @fixed("catch") class KwCatch extends Token
-  @fixed("class") class KwClass extends Token
-  @fixed("def") class KwDef extends Token
-  @fixed("do") class KwDo extends Token
-  @fixed("else") class KwElse extends Token
-  @fixed("enum") class KwEnum extends Token
-  @fixed("given") class KwGiven extends Token
-  @fixed("extends") class KwExtends extends Token
-  @fixed("false") class KwFalse extends Token
-  @fixed("final") class KwFinal extends Token
-  @fixed("finally") class KwFinally extends Token
-  @fixed("for") class KwFor extends Token
-  @fixed("forSome") class KwForsome extends Token
-  @fixed("if") class KwIf extends Token
-  @fixed("implicit") class KwImplicit extends Token
-  @fixed("import") class KwImport extends Token
-  @fixed("export") class KwExport extends Token
-  @fixed("lazy") class KwLazy extends Token
-  @fixed("match") class KwMatch extends Token
-  @fixed("macro") class KwMacro extends Token
-  @fixed("new") class KwNew extends Token
-  @fixed("null") class KwNull extends Token
-  @fixed("object") class KwObject extends Token
-  @fixed("override") class KwOverride extends Token
-  @fixed("package") class KwPackage extends Token
-  @fixed("private") class KwPrivate extends Token
-  @fixed("protected") class KwProtected extends Token
-  @fixed("return") class KwReturn extends Token
-  @fixed("sealed") class KwSealed extends Token
-  @fixed("super") class KwSuper extends Token
-  @fixed("this") class KwThis extends Token
-  @fixed("throw") class KwThrow extends Token
-  @fixed("trait") class KwTrait extends Token
-  @fixed("true") class KwTrue extends Token
-  @fixed("try") class KwTry extends Token
-  @fixed("type") class KwType extends Token
-  @fixed("val") class KwVal extends Token
-  @fixed("var") class KwVar extends Token
-  @fixed("while") class KwWhile extends Token
-  @fixed("with") class KwWith extends Token
-  @fixed("yield") class KwYield extends Token
-  @fixed("then") class KwThen extends Token
+  @fixed("abstract") class KwAbstract extends ModifierKeyword
+  @fixed("case") class KwCase extends Keyword
+  @fixed("catch") class KwCatch extends Keyword
+  @fixed("class") class KwClass extends Keyword
+  @fixed("def") class KwDef extends Keyword
+  @fixed("do") class KwDo extends Keyword
+  @fixed("else") class KwElse extends Keyword
+  @fixed("enum") class KwEnum extends Keyword
+  @fixed("export") class KwExport extends Keyword
+  @fixed("extends") class KwExtends extends Keyword
+  @fixed("false") class KwFalse extends Literal
+  @fixed("final") class KwFinal extends ModifierKeyword
+  @fixed("finally") class KwFinally extends Keyword
+  @fixed("for") class KwFor extends Keyword
+  @fixed("forSome") class KwForsome extends Keyword
+  @fixed("given") class KwGiven extends Keyword
+  @fixed("if") class KwIf extends Keyword
+  @fixed("implicit") class KwImplicit extends ModifierKeyword
+  @fixed("import") class KwImport extends Keyword
+  @fixed("lazy") class KwLazy extends ModifierKeyword
+  @fixed("match") class KwMatch extends Keyword
+  @fixed("macro") class KwMacro extends Keyword
+  @fixed("new") class KwNew extends Keyword
+  @fixed("null") class KwNull extends Literal
+  @fixed("object") class KwObject extends Keyword
+  @fixed("override") class KwOverride extends ModifierKeyword
+  @fixed("package") class KwPackage extends Keyword
+  @fixed("private") class KwPrivate extends ModifierKeyword
+  @fixed("protected") class KwProtected extends ModifierKeyword
+  @fixed("return") class KwReturn extends Keyword
+  @fixed("sealed") class KwSealed extends ModifierKeyword
+  @fixed("super") class KwSuper extends Keyword
+  @fixed("then") class KwThen extends Keyword
+  @fixed("this") class KwThis extends Keyword
+  @fixed("throw") class KwThrow extends Keyword
+  @fixed("trait") class KwTrait extends Keyword
+  @fixed("true") class KwTrue extends Literal
+  @fixed("try") class KwTry extends Keyword
+  @fixed("type") class KwType extends Keyword
+  @fixed("val") class KwVal extends Keyword
+  @fixed("var") class KwVar extends Keyword
+  @fixed("while") class KwWhile extends Keyword
+  @fixed("with") class KwWith extends Keyword
+  @fixed("yield") class KwYield extends Keyword
 
   // Symbolic keywords
-  @fixed("#") class Hash extends Token
-  @fixed(":") class Colon extends Token
-  @fixed("<%") class Viewbound extends Token
-  @freeform("<-") class LeftArrow extends Token
-  @fixed("<:") class Subtype extends Token
-  @fixed("=") class Equals extends Token
-  @freeform("=>") class RightArrow extends Token
-  @fixed(">:") class Supertype extends Token
-  @fixed("@") class At extends Token
-  @fixed("_") class Underscore extends Token
-  @fixed("=>>") class TypeLambdaArrow extends Token
-  @fixed("?=>") class ContextArrow extends Token
-  @fixed("'") class MacroQuote extends Token
-  @fixed("$") class MacroSplice extends Token
+  @fixed("#") class Hash extends SymbolicKeyword
+  @fixed(":") class Colon extends SymbolicKeyword
+  @fixed("<%") class Viewbound extends SymbolicKeyword
+  @freeform("<-") class LeftArrow extends SymbolicKeyword
+  @fixed("<:") class Subtype extends SymbolicKeyword
+  @fixed("=") class Equals extends SymbolicKeyword
+  @freeform("=>") class RightArrow extends SymbolicKeyword
+  @fixed(">:") class Supertype extends SymbolicKeyword
+  @fixed("@") class At extends SymbolicKeyword
+  @fixed("_") class Underscore extends SymbolicKeyword
+  @fixed("=>>") class TypeLambdaArrow extends SymbolicKeyword
+  @fixed("?=>") class ContextArrow extends SymbolicKeyword
+  @fixed("'") class MacroQuote extends SymbolicKeyword
+  @fixed("$") class MacroSplice extends SymbolicKeyword
 
   // Delimiters
-  @fixed("(") class LeftParen extends Token
-  @fixed(")") class RightParen extends Token
-  @fixed(",") class Comma extends Token
-  @fixed(".") class Dot extends Token
-  @fixed(";") class Semicolon extends Token
-  @fixed("[") class LeftBracket extends Token
-  @fixed("]") class RightBracket extends Token
-  @fixed("{") class LeftBrace extends Token
-  @fixed("}") class RightBrace extends Token
+  @fixed("(") class LeftParen extends OpenDelim
+  @fixed(")") class RightParen extends CloseDelim
+  @fixed(",") class Comma extends Punct
+  @fixed(".") class Dot extends Punct
+  @fixed(";") class Semicolon extends Punct
+  @fixed("[") class LeftBracket extends OpenDelim
+  @fixed("]") class RightBracket extends CloseDelim
+  @fixed("{") class LeftBrace extends OpenDelim
+  @fixed("}") class RightBrace extends CloseDelim
 
-  // Literals (include some keywords from above, constants, interpolations and xml)
   object Constant {
-    @freeform("integer constant") class Int(value: scala.BigInt) extends Token
-    @freeform("long constant") class Long(value: scala.BigInt) extends Token
-    @freeform("float constant") class Float(value: scala.BigDecimal) extends Token
-    @freeform("double constant") class Double(value: scala.BigDecimal) extends Token
-    @freeform("character constant") class Char(value: scala.Char) extends Token
-    @freeform("symbol constant") class Symbol(value: scala.Symbol) extends Token
-    @freeform("string constant") class String(value: Predef.String) extends Token
+    @freeform("integer constant") class Int(value: BigInt) extends NumericConstant[BigInt]
+    @freeform("long constant") class Long(value: BigInt) extends NumericConstant[BigInt]
+    @freeform("float constant") class Float(value: BigDecimal) extends NumericConstant[BigDecimal]
+    @freeform("double constant") class Double(value: BigDecimal) extends NumericConstant[BigDecimal]
+    @freeform("character constant") class Char(value: scala.Char) extends Constant[scala.Char]
+    @freeform("symbol constant") class Symbol(value: scala.Symbol) extends Constant[scala.Symbol]
+    @freeform("string constant") class String(value: Predef.String) extends Constant[Predef.String]
   }
   // NOTE: Here's example tokenization of q"${foo}bar".
   // BOF, Id(q)<"q">, Start<"\"">, Part("")<"">, SpliceStart<"$">, {, foo, }, SpliceEnd<"">, Part("bar")<"bar">, End("\""), EOF.
@@ -132,12 +154,12 @@ object Token {
   }
 
   // Trivia
-  @fixed(" ") class Space extends Token
-  @fixed("\t") class Tab extends Token
-  @fixed("\r") class CR extends Token
-  @fixed("\n") class LF extends Token
-  @fixed("\f") class FF extends Token
-  @freeform("comment") class Comment(value: String) extends Token
+  @fixed(" ") class Space extends HSpace
+  @fixed("\t") class Tab extends HSpace
+  @fixed("\r") class CR extends AtEOL
+  @fixed("\n") class LF extends EOL
+  @fixed("\f") class FF extends EOL
+  @freeform("comment") class Comment(value: String) extends Trivia
   @freeform("beginning of file") class BOF extends Token {
     def this(input: Input, dialect: Dialect) = this(input, dialect, 0)
     def end = start
@@ -149,7 +171,7 @@ object Token {
 
   // NOTE: in order to maintain conceptual compatibility with scala.reflect's implementation,
   // Ellipsis.rank = 1 means .., Ellipsis.rank = 2 means ..., etc
-  @freeform("\n\n") private[meta] class LFLF extends Token
+  @freeform("\n\n") private[meta] class LFLF extends AtEOL
   @freeform("ellipsis") private[meta] class Ellipsis(rank: Int) extends Token
   @freeform("unquote") private[meta] class Unquote extends Token
 


Test patch:
diff --git a/tests/jvm/src/test/scala-2.13/scala/meta/tests/api/SurfaceSuite.scala b/tests/jvm/src/test/scala-2.13/scala/meta/tests/api/SurfaceSuite.scala
index 65d2585b0..d125b6e93 100644
--- a/tests/jvm/src/test/scala-2.13/scala/meta/tests/api/SurfaceSuite.scala
+++ b/tests/jvm/src/test/scala-2.13/scala/meta/tests/api/SurfaceSuite.scala
@@ -149,7 +149,6 @@ class SurfaceSuite extends FunSuite {
       |scala.meta.tokenizers.Tokenized.Success *
       |scala.meta.tokens
       |scala.meta.tokens.Token
-      |scala.meta.tokens.Token.Constant *
       |scala.meta.tokens.Token.Indentation *
       |scala.meta.tokens.Token.Interpolation *
       |scala.meta.tokens.Token.Xml *
diff --git a/tests/jvm/src/test/scala/scala/meta/tests/tokens/ReflectionSuite.scala b/tests/jvm/src/test/scala/scala/meta/tests/tokens/ReflectionSuite.scala
index 59a86c02c..917f97fd4 100644
--- a/tests/jvm/src/test/scala/scala/meta/tests/tokens/ReflectionSuite.scala
+++ b/tests/jvm/src/test/scala/scala/meta/tests/tokens/ReflectionSuite.scala
@@ -18,7 +18,7 @@ class ReflectionSuite extends FunSuite {
 
   test("freeform tokens") {
     assertNoDiff(
-      tokens.filter(_.isFreeform).map(_.prefix).mkString(EOL),
+      tokens.filter(_.isFreeform).map(_.prefix).sorted.mkString(EOL),
       """
       |Token.BOF
       |Token.Comment
diff --git a/tests/shared/src/test/scala/scala/meta/tests/prettyprinters/PublicSuite.scala b/tests/shared/src/test/scala/scala/meta/tests/prettyprinters/PublicSuite.scala
index 631d32f3d..26ae6da86 100644
--- a/tests/shared/src/test/scala/scala/meta/tests/prettyprinters/PublicSuite.scala
+++ b/tests/shared/src/test/scala/scala/meta/tests/prettyprinters/PublicSuite.scala
@@ -407,10 +407,6 @@ class PublicSuite extends FunSuite {
     assert(tokens.syntax == "foo + bar")
   }
 
-  test("scala.meta.tokens.Token.Constant.toString") {
-    // n/a
-  }
-
   test("scala.meta.tokens.Token.Interpolation.toString") {
     // n/a
   }