Repository: scalameta/scalameta
Pull request #3226 (2023-07-04 09:48:46)

CommunityDottySuite: lazily traverse repository
Existing recursive implementation would result in `too many open files` on sufficiently large repositories.

kitbellew: @tgodzik i need to revise this. looked at community build check, shows files parsed as 0.

Patch:
diff --git a/scalameta/parsers/shared/src/main/scala/scala/meta/internal/parsers/ScannerTokens.scala b/scalameta/parsers/shared/src/main/scala/scala/meta/internal/parsers/ScannerTokens.scala
index 95948d86c..d03926a15 100644
--- a/scalameta/parsers/shared/src/main/scala/scala/meta/internal/parsers/ScannerTokens.scala
+++ b/scalameta/parsers/shared/src/main/scala/scala/meta/internal/parsers/ScannerTokens.scala
@@ -574,7 +574,7 @@ final class ScannerTokens(val tokens: Tokens)(implicit dialect: Dialect) {
           case soft.KwDerives() => getTemplateInherit(sepRegions)
           case soft.KwExtension()
               if next.isAny[LeftParen, LeftBracket] &&
-                prevToken.isAny[StatSep, Indentation, LeftBrace] =>
+                prevToken.isAny[StatSep, Indentation, LeftBrace, RightArrow] =>
             currRef(RegionExtensionMark :: sepRegions)
           case _ => currRef(sepRegions)
         }


Test patch:
diff --git a/community-test/src/test/scala/CommunityDottySuite.scala b/community-test/src/test/scala/CommunityDottySuite.scala
index 8c78769a8..47d7cdd6c 100644
--- a/community-test/src/test/scala/CommunityDottySuite.scala
+++ b/community-test/src/test/scala/CommunityDottySuite.scala
@@ -16,7 +16,7 @@ class CommunityDottySuite extends FunSuite {
 
   val communityDirectory = Paths.get("community-projects")
 
-  def fetchCommunityBuild(build: CommunityBuild): Unit = {
+  def fetchCommunityBuild(build: CommunityBuild): Path = {
     if (!Files.exists(communityDirectory)) Files.createDirectory(communityDirectory)
 
     val folderPath = communityDirectory.resolve(build.name)
@@ -34,6 +34,8 @@ class CommunityDottySuite extends FunSuite {
 
       assert(clue(result) == 0, s"Checking out community build ${build.name} failed")
     }
+
+    folderPath
   }
 
   case class CommunityBuild(
@@ -41,6 +43,7 @@ class CommunityDottySuite extends FunSuite {
       commit: String,
       name: String,
       excluded: List[String],
+      checkedFiles: Int,
       dialect: Dialect,
       dialectAlt: Option[Dialect] = None
   )
@@ -64,12 +67,12 @@ class CommunityDottySuite extends FunSuite {
     )
 
   val communityBuilds = List(
-    dottyBuild("3.0.2", dialects.Scala30),
-    dottyBuild("3.1.3", dialects.Scala31),
-    dottyBuild("3.2.2", dialects.Scala32),
-    dottyBuild("3.3.1-RC1", dialects.Scala33),
+    dottyBuild("3.0.2", dialects.Scala30, 886),
+    dottyBuild("3.1.3", dialects.Scala31, 946),
+    dottyBuild("3.2.2", dialects.Scala32, 996),
+    dottyBuild("3.3.1-RC1", dialects.Scala33, 1028),
     // latest commit from 30.03.2021
-    munitBuild("06346adfe3519c384201eec531762dad2f4843dc", dialects.Scala213)
+    munitBuild("06346adfe3519c384201eec531762dad2f4843dc", dialects.Scala213, 102)
   )
 
   for (build <- communityBuilds) {
@@ -79,9 +82,9 @@ class CommunityDottySuite extends FunSuite {
   }
 
   def check(implicit build: CommunityBuild): Unit = {
-    fetchCommunityBuild(build)
+    val folder = fetchCommunityBuild(build)
 
-    val stats = checkFilesRecursive(communityDirectory.resolve(build.name))
+    val stats = checkFilesRecursive(folder.toAbsolutePath).fold(InitTestStats)(merger)
     val timePer1KLines = Math.round(stats.timeTaken / (stats.linesParsed / 1000.0))
 
     println("--------------------------")
@@ -96,6 +99,9 @@ class CommunityDottySuite extends FunSuite {
     println(s"Parsing speed per 1k lines ===> ${timePer1KLines} ms/1klines")
     println("--------------------------")
     stats.lastError.foreach(e => throw e)
+
+    assertEquals(stats.errors, 0)
+    assertEquals(stats.checkedFiles, build.checkedFiles)
   }
 
   def timeIt(block: => Unit): Long = {
@@ -105,25 +111,22 @@ class CommunityDottySuite extends FunSuite {
     t1 - t0
   }
 
-  def checkFilesRecursive(parent: Path)(implicit build: CommunityBuild): TestStats = {
-    val absPath = parent.toAbsolutePath
-    val absPathString = absPath.toString
-    if (ignoreParts.exists(absPathString.contains))
-      return InitTestStats
-    if (Files.isDirectory(parent)) {
-      import scala.collection.JavaConverters._
-      Files
-        .list(parent)
-        .map(checkFilesRecursive)
-        .iterator()
-        .asScala
-        .fold(InitTestStats)(merger)
-    } else {
-      if (absPathString.endsWith(".scala")) {
-        checkAbsPath(absPath, absPathString)
-      } else InitTestStats
+  private def checkFilesRecursive(path: Path)(implicit cb: CommunityBuild): Iterator[TestStats] =
+    if (ignoreParts.exists(path.endsWith)) Iterator.empty
+    else {
+      val ds = Files.newDirectoryStream(path)
+      val (dirs, files) =
+        try {
+          import scala.collection.JavaConverters._
+          ds.iterator().asScala.toList.partition(Files.isDirectory(_))
+        } finally {
+          ds.close()
+        }
+      files.iterator.flatMap { x =>
+        val fileStr = x.toString
+        if (fileStr.endsWith(".scala")) Some(checkAbsPath(x, fileStr)) else None
+      } ++ dirs.iterator.flatMap(checkFilesRecursive)
     }
-  }
 
   def checkAbsPath(absPath: Path, absPathString: String)(
       implicit build: CommunityBuild
@@ -167,21 +170,20 @@ class CommunityDottySuite extends FunSuite {
     build.excluded.exists(el => path.endsWith(el))
   }
 
-  private def dottyBuild(ref: String, dialect: Dialect): CommunityBuild = {
-    val exclude = Nil
-    CommunityBuild("https://github.com/lampepfl/dotty.git", ref, "dotty", exclude, dialect)
+  private def dottyBuild(ref: String, dialect: Dialect, files: Int): CommunityBuild = {
+    CommunityBuild("https://github.com/lampepfl/dotty.git", ref, "dotty", Nil, files, dialect)
   }
 
-  private def munitBuild(ref: String, dialect: Dialect): CommunityBuild = {
-    val exclude = Nil
-    CommunityBuild("https://github.com/scalameta/munit.git", ref, "munit", exclude, dialect)
+  private def munitBuild(ref: String, dialect: Dialect, files: Int): CommunityBuild = {
+    CommunityBuild("https://github.com/scalameta/munit.git", ref, "munit", Nil, files, dialect)
   }
 
   final val ignoreParts = List(
-    "/tests/",
-    "/test-resources/scripting/",
-    "/test-resources/repl/",
-    "/sbt-test/",
-    "/out/"
-  )
+    ".git/",
+    "tests/",
+    "test-resources/scripting/",
+    "test-resources/repl/",
+    "sbt-test/",
+    "out/"
+  ).map(Paths.get(_))
 }
diff --git a/tests/shared/src/test/scala/scala/meta/tests/parsers/dotty/ExtensionMethodsSuite.scala b/tests/shared/src/test/scala/scala/meta/tests/parsers/dotty/ExtensionMethodsSuite.scala
index 916573a73..11c4b372e 100644
--- a/tests/shared/src/test/scala/scala/meta/tests/parsers/dotty/ExtensionMethodsSuite.scala
+++ b/tests/shared/src/test/scala/scala/meta/tests/parsers/dotty/ExtensionMethodsSuite.scala
@@ -704,4 +704,120 @@ class ExtensionMethodsSuite extends BaseDottySuite {
     )
   }
 
+  test("#3231 1") {
+    runTestAssert[Stat](
+      """|object A {
+         |  self =>
+         |  extension (x: X)
+         |    @annoFoo
+         |    def foo: Foo =
+         |      getFoo
+         |
+         |    @annoBar
+         |    def bar: Bar =
+         |      getBar
+         |}
+         |""".stripMargin,
+      assertLayout = Some(
+        """|object A { self =>
+           |  extension (x: X){
+           |    @annoFoo def foo: Foo = getFoo
+           |    @annoBar def bar: Bar = getBar
+           |  }
+           |}
+           |""".stripMargin
+      )
+    )(
+      Defn.Object(
+        Nil,
+        tname("A"),
+        Template(
+          Nil,
+          Nil,
+          Self(tname("self"), None),
+          Defn.ExtensionGroup(
+            Nil,
+            List(List(tparam("x", "X"))),
+            Term.Block(
+              List(
+                Defn.Def(
+                  List(Mod.Annot(Init(pname("annoFoo"), Name.Anonymous(), Nil))),
+                  tname("foo"),
+                  Nil,
+                  Some(pname("Foo")),
+                  tname("getFoo")
+                ),
+                Defn.Def(
+                  List(Mod.Annot(Init(pname("annoBar"), Name.Anonymous(), Nil))),
+                  tname("bar"),
+                  Nil,
+                  Some(pname("Bar")),
+                  tname("getBar")
+                )
+              )
+            )
+          ) :: Nil,
+          Nil
+        )
+      )
+    )
+  }
+
+  test("#3231 2") {
+    runTestAssert[Stat](
+      """|object A {
+         |  self =>
+         |  extension (x: X)
+         |    private def foo: Foo =
+         |      getFoo
+         |
+         |    protected def bar: Bar =
+         |      getBar
+         |}
+         |""".stripMargin,
+      assertLayout = Some(
+        """|object A { self =>
+           |  extension (x: X){
+           |    private def foo: Foo = getFoo
+           |    protected def bar: Bar = getBar
+           |  }
+           |}
+           |""".stripMargin
+      )
+    )(
+      Defn.Object(
+        Nil,
+        tname("A"),
+        Template(
+          Nil,
+          Nil,
+          Self(tname("self"), None),
+          Defn.ExtensionGroup(
+            Nil,
+            List(List(tparam("x", "X"))),
+            Term.Block(
+              List(
+                Defn.Def(
+                  List(Mod.Private(Name.Anonymous())),
+                  tname("foo"),
+                  Nil,
+                  Some(pname("Foo")),
+                  tname("getFoo")
+                ),
+                Defn.Def(
+                  List(Mod.Protected(Name.Anonymous())),
+                  tname("bar"),
+                  Nil,
+                  Some(pname("Bar")),
+                  tname("getBar")
+                )
+              )
+            )
+          ) :: Nil,
+          Nil
+        )
+      )
+    )
+  }
+
 }