Repository: joernio/joern (scala)

Issue:
jssrc: ignore transpiled files if original source files get scanned
There are repos like https://github.com/google/closure-templates/tree/master/javascript that have transpiled files and original source files next to each other. Scanning that with standard settings will result in both of these files in the CPG.

We can consider that user error, but maybe we can add some simple logic to drop files with a source map when the original source file is scanned at the same time?

Pull request:
Fixes: https://github.com/joernio/joern/issues/2547

Patch:
diff --git a/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/utils/AstGenRunner.scala b/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/utils/AstGenRunner.scala
index a4f4db3fad7f..7348dc8df269 100644
--- a/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/utils/AstGenRunner.scala
+++ b/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/utils/AstGenRunner.scala
@@ -23,7 +23,7 @@ object AstGenRunner {
 
   private val LineLengthThreshold: Int = 10000
 
-  private val TypeDefinitionFileExtensions = List(".t.ts.json", ".d.ts.json")
+  private val TypeDefinitionFileExtensions = List(".t.ts", ".d.ts")
 
   private val MinifiedPathRegex: Regex = ".*([.-]min\\..*js|bundle\\.js)".r
 
@@ -128,15 +128,14 @@ class AstGenRunner(config: Config) {
     skipped.flatten
   }
 
-  private def isIgnoredByUserConfig(filePath: String, config: Config, out: File): Boolean = {
-    val resolvedFilePath = filePath.stripSuffix(".json").replace(out.pathAsString, config.inputPath)
+  private def isIgnoredByUserConfig(filePath: String): Boolean = {
     lazy val isInIgnoredFiles = config.ignoredFiles.exists {
-      case ignorePath if File(ignorePath).isDirectory => resolvedFilePath.startsWith(ignorePath)
-      case ignorePath                                 => resolvedFilePath == ignorePath
+      case ignorePath if File(ignorePath).isDirectory => filePath.startsWith(ignorePath)
+      case ignorePath                                 => filePath == ignorePath
     }
-    lazy val isInIgnoredFileRegex = config.ignoredFilesRegex.matches(resolvedFilePath)
+    lazy val isInIgnoredFileRegex = config.ignoredFilesRegex.matches(filePath)
     if (isInIgnoredFiles || isInIgnoredFileRegex) {
-      logger.debug(s"'$resolvedFilePath' ignored by user configuration")
+      logger.debug(s"'$filePath' ignored by user configuration")
       true
     } else {
       false
@@ -156,42 +155,70 @@ class AstGenRunner(config: Config) {
     case _ => false
   }
 
-  private def isIgnoredByDefault(filePath: String, config: Config, out: File): Boolean = {
-    val resolvedFilePath   = filePath.stripSuffix(".json").replace(out.pathAsString, config.inputPath)
-    lazy val isIgnored     = IgnoredFilesRegex.exists(_.matches(resolvedFilePath))
-    lazy val isIgnoredTest = IgnoredTestsRegex.exists(_.matches(resolvedFilePath))
-    lazy val isMinified    = isMinifiedFile(resolvedFilePath)
+  private def isIgnoredByDefault(filePath: String): Boolean = {
+    lazy val isIgnored     = IgnoredFilesRegex.exists(_.matches(filePath))
+    lazy val isIgnoredTest = IgnoredTestsRegex.exists(_.matches(filePath))
+    lazy val isMinified    = isMinifiedFile(filePath)
     if (isIgnored || isIgnoredTest || isMinified) {
-      logger.debug(s"'$resolvedFilePath' ignored by default")
+      logger.debug(s"'$filePath' ignored by default")
       true
     } else {
       false
     }
   }
 
-  private def filterFiles(files: List[String], config: Config, out: File): List[String] = {
-    files.filter {
-      // We are not interested in JS / TS type definition files at this stage.
-      // TODO: maybe we can enable that later on and use the type definitions there
-      //  for enhancing the CPG with additional type information for functions
-      case filePath if TypeDefinitionFileExtensions.exists(filePath.endsWith) => false
-      case filePath if isIgnoredByUserConfig(filePath, config, out)           => false
-      case filePath if isIgnoredByDefault(filePath, config, out)              => false
-      case _                                                                  => true
+  private def isTranspiledFile(filePath: String): Boolean = {
+    val file = File(filePath)
+    // We ignore files iff:
+    // - they are *.js files and
+    // - they contain a //sourceMappingURL comment or have an associated source map file and
+    // - a file with the same name is located directly next to them
+    lazy val isJsFile            = file.exists && file.extension.contains(".js")
+    lazy val hasSourceMapComment = IOUtils.readLinesInFile(file.path).exists(_.contains("//sourceMappingURL"))
+    lazy val hasSourceMapFile    = File(s"$filePath.map").exists
+    lazy val hasSourceMap        = hasSourceMapComment || hasSourceMapFile
+    lazy val hasFileWithSameName =
+      file.siblings.exists(_.nameWithoutExtension(includeAll = false) == file.nameWithoutExtension)
+    if (isJsFile && hasSourceMap && hasFileWithSameName) {
+      logger.debug(s"'$filePath' ignored by default (seems to be the result of transpilation)")
+      true
+    } else {
+      false
+    }
+
+  }
+
+  private def filterFiles(files: List[String], out: File): List[String] = {
+    files.filter { file =>
+      file.stripSuffix(".json").replace(out.pathAsString, config.inputPath) match {
+        // We are not interested in JS / TS type definition files at this stage.
+        // TODO: maybe we can enable that later on and use the type definitions there
+        //  for enhancing the CPG with additional type information for functions
+        case filePath if TypeDefinitionFileExtensions.exists(filePath.endsWith) => false
+        case filePath if isIgnoredByUserConfig(filePath)                        => false
+        case filePath if isIgnoredByDefault(filePath)                           => false
+        case filePath if isTranspiledFile(filePath)                             => false
+        case _                                                                  => true
+      }
     }
   }
 
   private def processEjsFiles(in: File, out: File, ejsFiles: List[String]): Try[Seq[String]] = {
     val tmpJsFiles = ejsFiles.map { ejsFilePath =>
-      val ejsFile           = File(ejsFilePath)
-      val ls                = SourceFiles.retrieveLineSeparator(ejsFilePath)
-      val sourceFileContent = IOUtils.readLinesInFile(ejsFile.path).mkString("", ls, ls)
-      val preprocessContent = new EjsPreprocessor().preprocess(sourceFileContent)
-      (out / in.relativize(ejsFile).toString).parent.createDirectoryIfNotExists(createParents = true)
-      val newEjsFile = ejsFile.copyTo(out / in.relativize(ejsFile).toString)
-      val jsFile     = newEjsFile.changeExtensionTo(".js").writeText(preprocessContent)
-      newEjsFile.createFile().writeText(sourceFileContent)
-      jsFile
+      val ejsFile             = File(ejsFilePath)
+      val maybeTranspiledFile = File(ejsFilePath.stripSuffix(".ejs") + ".js")
+      if (isTranspiledFile(maybeTranspiledFile.pathAsString)) {
+        maybeTranspiledFile
+      } else {
+        val ls                = SourceFiles.retrieveLineSeparator(ejsFilePath)
+        val sourceFileContent = IOUtils.readLinesInFile(ejsFile.path).mkString("", ls, ls)
+        val preprocessContent = new EjsPreprocessor().preprocess(sourceFileContent)
+        (out / in.relativize(ejsFile).toString).parent.createDirectoryIfNotExists(createParents = true)
+        val newEjsFile = ejsFile.copyTo(out / in.relativize(ejsFile).toString)
+        val jsFile     = newEjsFile.changeExtensionTo(".js").writeText(preprocessContent)
+        newEjsFile.createFile().writeText(sourceFileContent)
+        jsFile
+      }
     }
 
     val result = ExternalCommand.run(s"$astGenCommand$executableArgs -t ts -o $out", out.toString())
@@ -238,7 +265,7 @@ class AstGenRunner(config: Config) {
     logger.info(s"Running astgen in '$in' ...")
     runAstGenNative(in, out) match {
       case Success(result) =>
-        val parsed  = filterFiles(SourceFiles.determine(out.toString(), Set(".json")), config, out)
+        val parsed  = filterFiles(SourceFiles.determine(out.toString(), Set(".json")), out)
         val skipped = skippedFiles(in, result.toList)
         AstGenRunnerResult(parsed.map((in.toString(), _)), skipped.map((in.toString(), _)))
       case Failure(f) =>


Test patch:
diff --git a/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/io/TranspiledFileDetectionTest.scala b/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/io/TranspiledFileDetectionTest.scala
new file mode 100644
index 000000000000..9c745fd4d236
--- /dev/null
+++ b/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/io/TranspiledFileDetectionTest.scala
@@ -0,0 +1,100 @@
+package io.joern.jssrc2cpg.io
+
+import better.files.File
+import io.joern.jssrc2cpg.Config
+import io.joern.jssrc2cpg.passes.AstCreationPass
+import io.joern.jssrc2cpg.utils.AstGenRunner
+import io.joern.x2cpg.X2Cpg.newEmptyCpg
+import io.shiftleft.semanticcpg.language._
+import org.scalatest.matchers.should.Matchers
+import org.scalatest.wordspec.AnyWordSpec
+
+class TranspiledFileDetectionTest extends AnyWordSpec with Matchers {
+
+  "Detecting transpiled files" should {
+    "skip transpiled files correctly (with source map comment)" in {
+      File.usingTemporaryDirectory("jssrc2cpgTest") { sourceDir =>
+        (sourceDir / "index.ts").createFile().writeText("console.log('Hello World!');")
+        (sourceDir / "index.js")
+          .createFile()
+          .writeText("""
+            |console.log('Hello World!');
+            |//sourceMappingURL=foo
+            |""".stripMargin)
+        val cpg          = newEmptyCpg()
+        val config       = Config(inputPath = sourceDir.toString, outputPath = sourceDir.toString, tsTypes = false)
+        val astGenResult = new AstGenRunner(config).execute(sourceDir)
+        new AstCreationPass(cpg, astGenResult, config).createAndApply()
+        cpg.file.name.l shouldBe List("index.ts")
+      }
+    }
+
+    "skip transpiled files correctly (with source map file)" in {
+      File.usingTemporaryDirectory("jssrc2cpgTest") { sourceDir =>
+        (sourceDir / "index.ts").createFile().writeText("console.log('Hello World!');")
+        (sourceDir / "index.js").createFile().writeText("console.log('Hello World!');")
+        (sourceDir / "index.js.map").createFile()
+        val cpg          = newEmptyCpg()
+        val config       = Config(inputPath = sourceDir.toString, outputPath = sourceDir.toString, tsTypes = false)
+        val astGenResult = new AstGenRunner(config).execute(sourceDir)
+        new AstCreationPass(cpg, astGenResult, config).createAndApply()
+        cpg.file.name.l shouldBe List("index.ts")
+      }
+    }
+
+    "skip transpiled files correctly (with other file types)" in {
+      File.usingTemporaryDirectory("jssrc2cpgTest") { sourceDir =>
+        (sourceDir / "index.vue").createFile().writeText("console.log('Hello World!');")
+        (sourceDir / "index.ejs").createFile().writeText("console.log('Hello World!');")
+        (sourceDir / "index.js").createFile().writeText("console.log('Hello World!');")
+        (sourceDir / "index.js.map").createFile()
+        val cpg          = newEmptyCpg()
+        val config       = Config(inputPath = sourceDir.toString, outputPath = sourceDir.toString, tsTypes = false)
+        val astGenResult = new AstGenRunner(config).execute(sourceDir)
+        new AstCreationPass(cpg, astGenResult, config).createAndApply()
+        cpg.file.name.l shouldBe List("index.ejs", "index.vue")
+      }
+    }
+
+    "not skip transpiled files when there is no source map file or comment" in {
+      File.usingTemporaryDirectory("jssrc2cpgTest") { sourceDir =>
+        (sourceDir / "index.ts").createFile().writeText("console.log('Hello World!');")
+        (sourceDir / "index.js").createFile().writeText("console.log('Hello World!');")
+        val cpg          = newEmptyCpg()
+        val config       = Config(inputPath = sourceDir.toString, outputPath = sourceDir.toString, tsTypes = false)
+        val astGenResult = new AstGenRunner(config).execute(sourceDir)
+        new AstCreationPass(cpg, astGenResult, config).createAndApply()
+        cpg.file.name.l shouldBe List("index.js", "index.ts")
+      }
+    }
+
+    "not skip transpiled files when there is no sibling file with same name but a source map file only" in {
+      File.usingTemporaryDirectory("jssrc2cpgTest") { sourceDir =>
+        (sourceDir / "index.js").createFile().writeText("console.log('Hello World!');")
+        (sourceDir / "index.js.map").createFile()
+        val cpg          = newEmptyCpg()
+        val config       = Config(inputPath = sourceDir.toString, outputPath = sourceDir.toString, tsTypes = false)
+        val astGenResult = new AstGenRunner(config).execute(sourceDir)
+        new AstCreationPass(cpg, astGenResult, config).createAndApply()
+        cpg.file.name.l shouldBe List("index.js")
+      }
+    }
+
+    "not skip transpiled files when there is no sibling file with same name but a source map comment only" in {
+      File.usingTemporaryDirectory("jssrc2cpgTest") { sourceDir =>
+        (sourceDir / "index.js")
+          .createFile()
+          .writeText("""
+              |console.log('Hello World!');
+              |//sourceMappingURL=foo
+              |""".stripMargin)
+        val cpg          = newEmptyCpg()
+        val config       = Config(inputPath = sourceDir.toString, outputPath = sourceDir.toString, tsTypes = false)
+        val astGenResult = new AstGenRunner(config).execute(sourceDir)
+        new AstCreationPass(cpg, astGenResult, config).createAndApply()
+        cpg.file.name.l shouldBe List("index.js")
+      }
+    }
+  }
+
+}