Repository: scala-steward-org/scala-steward (scala)

Issue:
do not check files/folders that are .gitignored
We're having some weird behaviour on our local Scala Steward instance and I started looking into it. Originally, the problem is that we're getting pull requests with invalid changes in them (I'll open another issue for that). But when investigating, I was trying to understand the codebase a little bit better and realized the following: Scala Steward looks into a bunch of files that are code-generated.

We're using smithy4s on our project and before compilation, we code generate Scala files into `target/scala/src_managed`. Scala steward will read those files and try to come up with `VersionPosition` or `ModulePosition`.

My suggestion is that we ignore and file/folder that is gitignored.

What do you think? Would you be open to review a PR that implement this?

Pull request:
Git has a .gitignore which can be used to exclude files from getting into the repository. It's pointless to scan files ignored via this mechanism as we'll fail to stage any changes to them.

This approach reads one .gitignore are the top of the file and then use it to check all the paths. This is not _exact_ but I think it's good enough.

I say that this is not exact because Git supports .gitignore file nested at arbitrary depth in the repository. To implement that, we'd need to re-implement the `walk` function which is not worth it in the scope of this issue.

Fixes https://github.com/scala-steward-org/scala-steward/issues/2979

Patch:
diff --git a/build.sbt b/build.sbt
index b2e63eb12f..879dc49b3f 100644
--- a/build.sbt
+++ b/build.sbt
@@ -121,6 +121,7 @@ lazy val core = myCrossProject("core")
       Dependencies.decline,
       Dependencies.fs2Core,
       Dependencies.fs2Io,
+      Dependencies.gitignore,
       Dependencies.http4sCirce,
       Dependencies.http4sClient,
       Dependencies.http4sCore,
diff --git a/modules/core/src/main/scala/org/scalasteward/core/edit/update/ScannerAlg.scala b/modules/core/src/main/scala/org/scalasteward/core/edit/update/ScannerAlg.scala
index 97acf020fb..3c1d9aa5e5 100644
--- a/modules/core/src/main/scala/org/scalasteward/core/edit/update/ScannerAlg.scala
+++ b/modules/core/src/main/scala/org/scalasteward/core/edit/update/ScannerAlg.scala
@@ -18,6 +18,8 @@ package org.scalasteward.core.edit.update
 
 import better.files.File
 import cats.effect.Concurrent
+import cats.syntax.functor._
+import com.github.arturopala.gitignore.GitIgnore
 import fs2.Stream
 import org.scalasteward.core.data.{Dependency, Repo, Version}
 import org.scalasteward.core.edit.update.data.{ModulePosition, VersionPosition}
@@ -56,19 +58,35 @@ final class ScannerAlg[F[_]](implicit
       .compile
       .foldMonoid
 
+  private def getGitIgnore(repoDir: File): F[Option[GitIgnore]] = {
+    val gitIgnore = repoDir / ".gitignore"
+    fileAlg.readFile(gitIgnore).map(_.map(GitIgnore.parse))
+  }
+
   private def findPathsContaining(
       repo: Repo,
       config: RepoConfig,
       string: String
-  ): Stream[F, FileData] =
-    Stream.eval(workspaceAlg.repoDir(repo)).flatMap { repoDir =>
-      val fileFilter = (file: File) => {
-        val path = repoDir.relativize(file).toString
-        val cond = !path.startsWith(".git/") &&
-          config.updates.fileExtensionsOrDefault.exists(path.endsWith)
-        Option.when(cond)(path)
-      }
-      val contentFilter = (content: String) => Some(content).filter(_.contains(string))
-      fileAlg.findFiles(repoDir, fileFilter, contentFilter).map(FileData.tupled)
+  ): Stream[F, FileData] = {
+    def fileFilter(repoDir: File, maybeGitIgnore: Option[GitIgnore]) = (file: File) => {
+      val path = repoDir.relativize(file).toString
+      val notDotGit = !path.startsWith(".git/")
+      val onlyKeepConfiguredExtensions =
+        config.updates.fileExtensionsOrDefault.exists(path.endsWith)
+      val allowedByGitIgnore = maybeGitIgnore.map(_.isAllowed(path)).getOrElse(true)
+      val cond = notDotGit &&
+        onlyKeepConfiguredExtensions &&
+        allowedByGitIgnore
+      Option.when(cond)(path)
     }
+    val contentFilter = (content: String) => Some(content).filter(_.contains(string))
+
+    for {
+      repoDir <- Stream.eval(workspaceAlg.repoDir(repo))
+      maybeRootGitIgnore <- Stream.eval(getGitIgnore(repoDir))
+      files <- fileAlg
+        .findFiles(repoDir, fileFilter(repoDir, maybeRootGitIgnore), contentFilter)
+        .map(FileData.tupled)
+    } yield files
+  }
 }
diff --git a/project/Dependencies.scala b/project/Dependencies.scala
index 9a944585f6..0e27c3d3ee 100644
--- a/project/Dependencies.scala
+++ b/project/Dependencies.scala
@@ -20,6 +20,7 @@ object Dependencies {
   val disciplineMunit = "org.typelevel" %% "discipline-munit" % "1.0.9"
   val fs2Core = "co.fs2" %% "fs2-core" % "3.6.1"
   val fs2Io = "co.fs2" %% "fs2-io" % fs2Core.revision
+  val gitignore = "com.github.arturopala" %% "gitignore" % "0.4.0"
   val http4sCore = "org.http4s" %% "http4s-core" % "1.0.0-M39"
   val http4sCirce = "org.http4s" %% "http4s-circe" % http4sCore.revision
   val http4sClient = "org.http4s" %% "http4s-client" % http4sCore.revision


Test patch:
diff --git a/modules/core/src/test/scala/org/scalasteward/core/edit/EditAlgTest.scala b/modules/core/src/test/scala/org/scalasteward/core/edit/EditAlgTest.scala
index d6487ae0e8..8e68975b46 100644
--- a/modules/core/src/test/scala/org/scalasteward/core/edit/EditAlgTest.scala
+++ b/modules/core/src/test/scala/org/scalasteward/core/edit/EditAlgTest.scala
@@ -27,21 +27,26 @@ class EditAlgTest extends FunSuite {
     val update = ("org.typelevel".g % "cats-core".a % "1.2.0" %> "1.3.0").single
     val file1 = repoDir / "build.sbt"
     val file2 = repoDir / "project/Dependencies.scala"
+    val gitignore = repoDir / ".gitignore"
 
     val state = MockState.empty
-      .addFiles(file1 -> """val catsVersion = "1.2.0"""", file2 -> "")
+      .addFiles(file1 -> """val catsVersion = "1.2.0"""", file2 -> "", gitignore -> "")
       .flatMap(editAlg.applyUpdate(data, update).runS)
       .unsafeRunSync()
 
     val expected = MockState.empty.copy(
       trace = Vector(
+        Cmd("read", gitignore.pathAsString),
         Cmd("test", "-f", repoDir.pathAsString),
+        Cmd("test", "-f", gitignore.pathAsString),
         Cmd("test", "-f", file1.pathAsString),
         Cmd("read", file1.pathAsString),
         Cmd("test", "-f", (repoDir / "project").pathAsString),
         Cmd("test", "-f", file2.pathAsString),
         Cmd("read", file2.pathAsString),
+        Cmd("read", gitignore.pathAsString),
         Cmd("test", "-f", repoDir.pathAsString),
+        Cmd("test", "-f", gitignore.pathAsString),
         Cmd("test", "-f", file1.pathAsString),
         Cmd("read", file1.pathAsString),
         Cmd("test", "-f", (repoDir / "project").pathAsString),
@@ -51,7 +56,7 @@ class EditAlgTest extends FunSuite {
         Cmd("write", file1.pathAsString),
         Cmd(gitStatus(repoDir))
       ),
-      files = Map(file1 -> """val catsVersion = "1.3.0"""", file2 -> "")
+      files = Map(file1 -> """val catsVersion = "1.3.0"""", file2 -> "", gitignore -> "")
     )
 
     assertEquals(state, expected)
@@ -65,30 +70,47 @@ class EditAlgTest extends FunSuite {
     val data = RepoData(repo, cache, RepoConfig.empty)
     val repoDir = workspaceAlg.repoDir(repo).unsafeRunSync()
     val update = ("org.scalameta".g % "scalafmt-core".a % "2.0.0" %> "2.1.0").single
+    val gitignore = repoDir / ".gitignore"
     val scalafmtConf = repoDir / scalafmtConfName
     val scalafmtConfContent = """maxColumn = 100
                                 |version = 2.0.0
                                 |align.openParenCallSite = false
                                 |""".stripMargin
     val buildSbt = repoDir / "build.sbt"
+    val target = repoDir / "target"
+    // this file should not be read because it's under target which is git ignored
+    val targetScalaFile = target / "SomeFile.scala"
 
     val state = MockState.empty
-      .addFiles(scalafmtConf -> scalafmtConfContent, buildSbt -> "")
+      .addFiles(
+        scalafmtConf -> scalafmtConfContent,
+        buildSbt -> "",
+        gitignore -> "target/",
+        targetScalaFile -> ""
+      )
       .flatMap(editAlg.applyUpdate(data, update).runS)
       .unsafeRunSync()
 
     val expected = MockState.empty.copy(
       trace = Vector(
+        Cmd("read", gitignore.pathAsString),
         Cmd("test", "-f", repoDir.pathAsString),
+        Cmd("test", "-f", gitignore.pathAsString),
         Cmd("test", "-f", scalafmtConf.pathAsString),
         Cmd("read", scalafmtConf.pathAsString),
         Cmd("test", "-f", buildSbt.pathAsString),
         Cmd("read", buildSbt.pathAsString),
+        Cmd("test", "-f", target.pathAsString),
+        Cmd("test", "-f", targetScalaFile.pathAsString),
+        Cmd("read", gitignore.pathAsString),
         Cmd("test", "-f", repoDir.pathAsString),
+        Cmd("test", "-f", gitignore.pathAsString),
         Cmd("test", "-f", scalafmtConf.pathAsString),
         Cmd("read", scalafmtConf.pathAsString),
         Cmd("test", "-f", buildSbt.pathAsString),
         Cmd("read", buildSbt.pathAsString),
+        Cmd("test", "-f", target.pathAsString),
+        Cmd("test", "-f", targetScalaFile.pathAsString),
         Cmd("read", scalafmtConf.pathAsString),
         Cmd("write", scalafmtConf.pathAsString),
         Cmd(
@@ -109,7 +131,9 @@ class EditAlgTest extends FunSuite {
             |version = 2.1.0
             |align.openParenCallSite = false
             |""".stripMargin,
-        buildSbt -> ""
+        buildSbt -> "",
+        gitignore -> "target/",
+        targetScalaFile -> ""
       )
     )