Repository: joernio/joern
Pull request #703 (2021-11-10 09:20:24)

joern-parse: cleanup and simplification
Now that data flow analysis is sufficiently fast, let's always calculate DDGs instead of having users run into the situation where data flow analysis doesn't work because these edges have not been calculated. Also, let's get rid of the term "enhancing the graph" when we mean enhancing it in a very specific way, that is, by applying the default overlays. Finally, if a user just runs `./joern-parse`, show them how to use it.


Patch:
diff --git a/joern-cli/src/main/scala/io/joern/Cpg2Scpg.scala b/joern-cli/src/main/scala/io/joern/DefaultOverlays.scala
similarity index 76%
rename from joern-cli/src/main/scala/io/joern/Cpg2Scpg.scala
rename to joern-cli/src/main/scala/io/joern/DefaultOverlays.scala
index 9011b23a1..ff658a41a 100644
--- a/joern-cli/src/main/scala/io/joern/Cpg2Scpg.scala
+++ b/joern-cli/src/main/scala/io/joern/DefaultOverlays.scala
@@ -4,7 +4,7 @@ import io.joern.dataflowengineoss.layers.dataflows.{OssDataFlow, OssDataFlowOpti
 import io.shiftleft.semanticcpg.layers._
 import io.shiftleft.codepropertygraph.Cpg
 
-object Cpg2Scpg {
+object DefaultOverlays {
 
   val DEFAULT_CPG_IN_FILE = "cpg.bin"
 
@@ -13,17 +13,15 @@ object Cpg2Scpg {
     * turning the CPG into an SCPG.
     * @param storeFilename the filename of the cpg
     * */
-  def run(storeFilename: String, dataFlow: Boolean): Cpg = {
+  def create(storeFilename: String): Cpg = {
     val cpg = CpgBasedTool.loadFromOdb(storeFilename)
     val context = new LayerCreatorContext(cpg)
     new Base().run(context)
     new TypeRelations().run(context)
     new ControlFlow().run(context)
     new CallGraph().run(context)
-    if (dataFlow) {
-      val options = new OssDataFlowOptions()
-      new OssDataFlow(options).run(context)
-    }
+    val options = new OssDataFlowOptions()
+    new OssDataFlow(options).run(context)
     cpg
   }
 
diff --git a/joern-cli/src/main/scala/io/joern/JoernParse.scala b/joern-cli/src/main/scala/io/joern/JoernParse.scala
index 6d2d9eda2..a61855103 100644
--- a/joern-cli/src/main/scala/io/joern/JoernParse.scala
+++ b/joern-cli/src/main/scala/io/joern/JoernParse.scala
@@ -5,11 +5,47 @@ import io.shiftleft.codepropertygraph.generated.Languages
 import io.joern.console.{FrontendConfig, InstallConfig}
 import io.joern.console.cpgcreation.{cpgGeneratorForLanguage, guessLanguage}
 import io.joern.CpgBasedTool.newCpgCreatedString
-
 import scala.jdk.CollectionConverters._
 
 object JoernParse extends App {
 
+  val optionParser = new scopt.OptionParser[ParserConfig]("joern-parse") {
+    arg[String]("input")
+      .optional()
+      .text("source file or directory containing source files")
+      .action((x, c) => c.copy(inputPath = x))
+
+    opt[String]("out")
+      .text("output filename")
+      .action((x, c) => c.copy(outputCpgFile = x))
+
+    opt[String]("language")
+      .text("source language")
+      .action((x, c) => c.copy(language = x))
+
+    opt[Unit]("list-languages")
+      .text("list available language options")
+      .action((_, c) => c.copy(listLanguages = true))
+
+    opt[String]("namespaces")
+      .text("namespaces to include: comma separated string")
+      .action((x, c) => c.copy(namespaces = x.split(",").map(_.trim).toList))
+
+    note("Overlay application stage")
+
+    opt[Unit]("nooverlays")
+      .text("do not apply default overlays")
+      .action((x, c) => c.copy(enhance = false))
+    opt[Unit]("overlaysonly")
+      .text("Only apply default overlays")
+      .action((x, c) => c.copy(enhanceOnly = true))
+
+    note("Misc")
+    help("help").text("display this help message")
+
+    note(s"Args specified after the $ARGS_DELIMITER separator will be passed to the front-end verbatim")
+  }
+
   // Special string used to separate joern-parse opts from frontend-specific opts
   val ARGS_DELIMITER = "--frontend-args"
   val DEFAULT_CPG_OUT_FILE = "cpg.bin"
@@ -25,7 +61,17 @@ object JoernParse extends App {
       System.exit(1)
   }
 
-  def run(): Either[String, String] = {
+  private def splitArgs(): (List[String], List[String]) = {
+    args.indexOf(ARGS_DELIMITER) match {
+      case -1 => (args.toList, Nil)
+
+      case splitIdx =>
+        val (parseOpts, frontendOpts) = args.toList.splitAt(splitIdx)
+        (parseOpts, frontendOpts.tail) // Take the tail to ignore the delimiter
+    }
+  }
+
+  private def run(): Either[String, String] = {
     parseConfig(parserArgs) match {
       case Right(config) =>
         if (config.listLanguages) {
@@ -35,39 +81,33 @@ object JoernParse extends App {
             _ <- checkInputPath(config)
             language <- getLanguage(config)
             _ <- generateCpg(config, language)
-            _ <- enhanceCpg(config)
+            _ <- applyDefaultOverlays(config)
           } yield newCpgCreatedString(config.outputCpgFile)
 
       case Left(err) => Left(err)
     }
   }
 
-  def splitArgs(): (List[String], List[String]) = {
-    args.indexOf(ARGS_DELIMITER) match {
-      case -1 => (args.toList, Nil)
+  private def checkInputPath(config: ParserConfig): Either[String, Unit] = {
 
-      case splitIdx =>
-        val (parseOpts, frontendOpts) = args.toList.splitAt(splitIdx)
-        (parseOpts, frontendOpts.tail) // Take the tail to ignore the delimiter
-    }
-  }
-
-  def checkInputPath(config: ParserConfig): Either[String, Unit] = {
-    if (config.inputPath == "" || !File(config.inputPath).exists) {
+    if (config.inputPath == "") {
+      println(optionParser.usage)
+      Left("Input path required")
+    } else if (!File(config.inputPath).exists) {
       Left("Input path does not exist at `" + config.inputPath + "`, exiting.")
     } else {
       Right(())
     }
   }
 
-  def buildLanguageList(): String = {
+  private def buildLanguageList(): String = {
     val s = new StringBuilder()
     s ++= "Available languages (case insensitive):\n"
     s ++= Languages.ALL.asScala.map(lang => s"- ${lang.toLowerCase}").mkString("\n")
     s.toString()
   }
 
-  def getLanguage(config: ParserConfig): Either[String, String] = {
+  private def getLanguage(config: ParserConfig): Either[String, String] = {
     if (config.language.isEmpty) {
       val inputPath = config.inputPath
       guessLanguage(inputPath) match {
@@ -83,7 +123,7 @@ object JoernParse extends App {
     }
   }
 
-  def generateCpg(config: ParserConfig, language: String): Either[String, String] = {
+  private def generateCpg(config: ParserConfig, language: String): Either[String, String] = {
     if (config.enhanceOnly) {
       Right("No generation required")
     } else {
@@ -98,11 +138,11 @@ object JoernParse extends App {
     }
   }
 
-  def enhanceCpg(config: ParserConfig): Either[String, String] = {
+  private def applyDefaultOverlays(config: ParserConfig): Either[String, String] = {
     try {
-      println("[+] Creating code property graph")
+      println("[+] Applying default overlays")
       if (config.enhance) {
-        Cpg2Scpg.run(config.outputCpgFile, config.dataFlow).close()
+        DefaultOverlays.create(config.outputCpgFile).close()
       }
       Right("Code property graph generation successful")
     } catch {
@@ -114,55 +154,18 @@ object JoernParse extends App {
                           outputCpgFile: String = DEFAULT_CPG_OUT_FILE,
                           namespaces: List[String] = List.empty,
                           enhance: Boolean = true,
-                          dataFlow: Boolean = true,
                           enhanceOnly: Boolean = false,
                           language: String = "",
                           listLanguages: Boolean = false)
 
-  def parseConfig(parserArgs: List[String]): Either[String, ParserConfig] =
-    new scopt.OptionParser[ParserConfig]("joern-parse") {
-      arg[String]("input")
-        .optional()
-        .text("source file or directory containing source files")
-        .action((x, c) => c.copy(inputPath = x))
-
-      opt[String]("out")
-        .text("output filename")
-        .action((x, c) => c.copy(outputCpgFile = x))
-
-      opt[String]("language")
-        .text("source language")
-        .action((x, c) => c.copy(language = x))
-
-      opt[Unit]("list-languages")
-        .text("list available language options")
-        .action((_, c) => c.copy(listLanguages = true))
-
-      opt[String]("namespaces")
-        .text("namespaces to include: comma separated string")
-        .action((x, c) => c.copy(namespaces = x.split(",").map(_.trim).toList))
-
-      note("Enhancement stage")
-
-      opt[Unit]("noenhance")
-        .text("do not run enhancement stage")
-        .action((x, c) => c.copy(enhance = false))
-      opt[Unit]("enhanceonly")
-        .text("Only run the enhancement stage")
-        .action((x, c) => c.copy(enhanceOnly = true))
-      opt[Unit]("nodataflow")
-        .text("do not perform data flow analysis in enhancement stage")
-        .action((x, c) => c.copy(dataFlow = false))
-
-      note("Misc")
-      help("help").text("display this help message")
-
-      note(s"Args specified after the $ARGS_DELIMITER separator will be passed to the front-end verbatim")
-    }.parse(parserArgs, ParserConfig()) match {
+  private def parseConfig(parserArgs: List[String]): Either[String, ParserConfig] = {
+
+    optionParser.parse(parserArgs, ParserConfig()) match {
       case Some(config) => Right(config)
 
       case None =>
         Left("Could not parse command line options")
     }
+  }
 
 }


Test patch:
diff --git a/joern-cli/src/test/scala/io/shiftleft/joern/AbstractJoernCliTest.scala b/joern-cli/src/test/scala/io/shiftleft/joern/AbstractJoernCliTest.scala
index fd9f50442..c17321352 100644
--- a/joern-cli/src/test/scala/io/shiftleft/joern/AbstractJoernCliTest.scala
+++ b/joern-cli/src/test/scala/io/shiftleft/joern/AbstractJoernCliTest.scala
@@ -21,7 +21,7 @@ trait AbstractJoernCliTest {
     val config = C2Cpg.Config(inputPaths = inputFilenames, outputPath = c2cpgOutFilename)
     c2cpg.runAndOutput(config).close()
     // Link CPG fragments and enhance to create semantic CPG
-    val cpg = Cpg2Scpg.run(c2cpgOutFilename, dataFlow = true)
+    val cpg = DefaultOverlays.create(c2cpgOutFilename)
     (cpg, c2cpgOutFilename)
   }