Repository: joernio/joern (scala)

Issue:
c2cpg: Code inside disabled preprocessor directive not parsed
While the code at https://github.com/joernio/joern/blob/master/joern-cli/frontends/c2cpg/src/main/scala/io/joern/c2cpg/parser/CdtParser.scala#L40 reads as if code inside disabled preprocessor directives should be parsed, this doesn't work for me.

Take, e.g., https://github.com/LibVNC/libvncserver - the code in `libvncclient/zrle.c` is not parsed when importing the directory with `importCode()`.

I would understand that this is the case if there is an #else directive but that's not the case for that example. 

Even better would be if we were able to define our own preprocessor directores like with the `-D` option for gcc/clang.

Pull request description:
`fromString` now optionally takes arguments and passes them to the individual frontend.
`importCode` already had this. Added tests for that.

Fixes: https://github.com/joernio/joern/issues/762

Patch:
diff --git a/console/build.sbt b/console/build.sbt
index 91bda85a95be..500777b728f4 100644
--- a/console/build.sbt
+++ b/console/build.sbt
@@ -10,9 +10,7 @@ val CirceVersion = "0.12.2"
 val AmmoniteVersion = "2.4.1"
 val ZeroturnaroundVersion = "1.13"
 
-dependsOn(
-  // Projects.macros,
-  Projects.fuzzyc2cpg % Test)
+dependsOn(Projects.fuzzyc2cpg % Test, Projects.c2cpg % Test)
 
 libraryDependencies ++= Seq(
   "io.shiftleft" %% "codepropertygraph" % Versions.cpg,
@@ -32,38 +30,39 @@ libraryDependencies ++= Seq(
 Test / packageBin / publishArtifact := true
 
 scalacOptions ++= Seq(
-  "-deprecation",                      // Emit warning and location for usages of deprecated APIs.
-  "-encoding", "utf-8",                // Specify character encoding used by source files.
-  "-explaintypes",                     // Explain type errors in more detail.
-  "-feature",                          // Emit warning and location for usages of features that should be imported explicitly.
-  "-language:existentials",            // Existential types (besides wildcard types) can be written and inferred
-  "-language:experimental.macros",     // Allow macro definition (besides implementation and application)
-  "-language:higherKinds",             // Allow higher-kinded types
-  "-language:implicitConversions",     // Allow definition of implicit functions called views
-  "-unchecked",                        // Enable additional warnings where generated code depends on assumptions.
-  "-Xcheckinit",                       // Wrap field accessors to throw an exception on uninitialized access.
-  "-Xlint:adapted-args",               // Warn if an argument list is modified to match the receiver.
-  "-Xlint:constant",                   // Evaluation of a constant arithmetic expression results in an error.
-  "-Xlint:delayedinit-select",         // Selecting member of DelayedInit.
-  "-Xlint:doc-detached",               // A Scaladoc comment appears to be detached from its element.
-  "-Xlint:inaccessible",               // Warn about inaccessible types in method signatures.
-  "-Xlint:infer-any",                  // Warn when a type argument is inferred to be `Any`.
-  "-Xlint:missing-interpolator",       // A string literal appears to be missing an interpolator id.
-  "-Xlint:option-implicit",            // Option.apply used implicit view.
-  "-Xlint:package-object-classes",     // Class or object defined in package object.
-  "-Xlint:poly-implicit-overload",     // Parameterized overloaded implicit methods are not visible as view bounds.
-  "-Xlint:private-shadow",             // A private field (or class parameter) shadows a superclass field.
-  "-Xlint:stars-align",                // Pattern sequence wildcard must align with sequence component.
-  "-Xlint:type-parameter-shadow",      // A local type parameter shadows a type already in scope.
-  "-Ywarn-dead-code",                  // Warn when dead code is identified.
-  "-Ywarn-extra-implicit",             // Warn when more than one implicit parameter section is defined.
-  "-Ywarn-numeric-widen",              // Warn when numerics are widened.
-  "-Ywarn-unused:implicits",           // Warn if an implicit parameter is unused.
-  "-Ywarn-unused:imports",             // Warn if an import selector is not referenced.
-  "-Ywarn-unused:locals",              // Warn if a local definition is unused.
-  "-Ywarn-unused:params",              // Warn if a value parameter is unused.
-  "-Ywarn-unused:patvars",             // Warn if a variable bound in a pattern is unused.
-  "-Ywarn-unused:privates",             // Warn if a private member is unused.
+  "-deprecation", // Emit warning and location for usages of deprecated APIs.
+  "-encoding",
+  "utf-8", // Specify character encoding used by source files.
+  "-explaintypes", // Explain type errors in more detail.
+  "-feature", // Emit warning and location for usages of features that should be imported explicitly.
+  "-language:existentials", // Existential types (besides wildcard types) can be written and inferred
+  "-language:experimental.macros", // Allow macro definition (besides implementation and application)
+  "-language:higherKinds", // Allow higher-kinded types
+  "-language:implicitConversions", // Allow definition of implicit functions called views
+  "-unchecked", // Enable additional warnings where generated code depends on assumptions.
+  "-Xcheckinit", // Wrap field accessors to throw an exception on uninitialized access.
+  "-Xlint:adapted-args", // Warn if an argument list is modified to match the receiver.
+  "-Xlint:constant", // Evaluation of a constant arithmetic expression results in an error.
+  "-Xlint:delayedinit-select", // Selecting member of DelayedInit.
+  "-Xlint:doc-detached", // A Scaladoc comment appears to be detached from its element.
+  "-Xlint:inaccessible", // Warn about inaccessible types in method signatures.
+  "-Xlint:infer-any", // Warn when a type argument is inferred to be `Any`.
+  "-Xlint:missing-interpolator", // A string literal appears to be missing an interpolator id.
+  "-Xlint:option-implicit", // Option.apply used implicit view.
+  "-Xlint:package-object-classes", // Class or object defined in package object.
+  "-Xlint:poly-implicit-overload", // Parameterized overloaded implicit methods are not visible as view bounds.
+  "-Xlint:private-shadow", // A private field (or class parameter) shadows a superclass field.
+  "-Xlint:stars-align", // Pattern sequence wildcard must align with sequence component.
+  "-Xlint:type-parameter-shadow", // A local type parameter shadows a type already in scope.
+  "-Ywarn-dead-code", // Warn when dead code is identified.
+  "-Ywarn-extra-implicit", // Warn when more than one implicit parameter section is defined.
+  "-Ywarn-numeric-widen", // Warn when numerics are widened.
+  "-Ywarn-unused:implicits", // Warn if an implicit parameter is unused.
+  "-Ywarn-unused:imports", // Warn if an import selector is not referenced.
+  "-Ywarn-unused:locals", // Warn if a local definition is unused.
+  "-Ywarn-unused:params", // Warn if a value parameter is unused.
+  "-Ywarn-unused:patvars", // Warn if a variable bound in a pattern is unused.
+  "-Ywarn-unused:privates", // Warn if a private member is unused.
   "-Yrangepos"
 )
 
diff --git a/console/src/main/scala/io/joern/console/cpgcreation/CCpgGenerator.scala b/console/src/main/scala/io/joern/console/cpgcreation/CCpgGenerator.scala
index 2778fc1b79be..361e9e1242f1 100644
--- a/console/src/main/scala/io/joern/console/cpgcreation/CCpgGenerator.scala
+++ b/console/src/main/scala/io/joern/console/cpgcreation/CCpgGenerator.scala
@@ -5,11 +5,11 @@ import io.joern.console.FrontendConfig
 import java.nio.file.Path
 
 /**
-  * Fuzzy C/C++ language frontend. Translates C/C++ source files
-  * into code property graphs via fuzzy parsing.
+  * C/C++ language frontend that translates C/C++ source files
+  * into code property graphs using Eclipse CDT parsing / preprocessing.
   * */
 case class CCpgGenerator(config: FrontendConfig, rootPath: Path) extends CpgGenerator {
-  lazy val command = rootPath.resolve("c2cpg.sh")
+  lazy val command: Path = rootPath.resolve("c2cpg.sh")
 
   /**
     * Generate a CPG for the given input path.
diff --git a/console/src/main/scala/io/joern/console/cpgcreation/CpgGeneratorFactory.scala b/console/src/main/scala/io/joern/console/cpgcreation/CpgGeneratorFactory.scala
index 4309a719d4a1..19a09fd4b1cf 100644
--- a/console/src/main/scala/io/joern/console/cpgcreation/CpgGeneratorFactory.scala
+++ b/console/src/main/scala/io/joern/console/cpgcreation/CpgGeneratorFactory.scala
@@ -10,6 +10,24 @@ import overflowdb.Config
 import java.nio.file.Path
 import scala.util.Try
 
+object CpgGeneratorFactory {
+  private val KNOWN_LANGUAGES = Set(
+    Languages.C,
+    Languages.CSHARP,
+    Languages.GOLANG,
+    Languages.GHIDRA,
+    Languages.JAVA,
+    Languages.JAVASCRIPT,
+    Languages.PYTHON,
+    Languages.FUZZY_TEST_LANG,
+    Languages.LLVM,
+    Languages.PHP,
+    Languages.KOTLIN,
+    Languages.NEWC,
+    Languages.JAVASRC
+  )
+}
+
 class CpgGeneratorFactory(config: ConsoleConfig) {
 
   /**
@@ -35,23 +53,7 @@ class CpgGeneratorFactory(config: ConsoleConfig) {
       }
   }
 
-  def languageIsKnown(language: String): Boolean = {
-    Set(
-      Languages.C,
-      Languages.CSHARP,
-      Languages.GOLANG,
-      Languages.GHIDRA,
-      Languages.JAVA,
-      Languages.JAVASCRIPT,
-      Languages.PYTHON,
-      Languages.FUZZY_TEST_LANG,
-      Languages.LLVM,
-      Languages.PHP,
-      Languages.KOTLIN,
-      Languages.NEWC,
-      Languages.JAVASRC
-    ).contains(language)
-  }
+  def languageIsKnown(language: String): Boolean = CpgGeneratorFactory.KNOWN_LANGUAGES.contains(language)
 
   def runGenerator(frontend: CpgGenerator,
                    inputPath: String,
diff --git a/console/src/main/scala/io/joern/console/cpgcreation/FuzzyCCpgGenerator.scala b/console/src/main/scala/io/joern/console/cpgcreation/FuzzyCCpgGenerator.scala
index 2cd3be1f9781..3e2120a14617 100644
--- a/console/src/main/scala/io/joern/console/cpgcreation/FuzzyCCpgGenerator.scala
+++ b/console/src/main/scala/io/joern/console/cpgcreation/FuzzyCCpgGenerator.scala
@@ -9,7 +9,7 @@ import java.nio.file.Path
   * into code property graphs via fuzzy parsing.
   * */
 case class FuzzyCCpgGenerator(config: FrontendConfig, rootPath: Path) extends CpgGenerator {
-  lazy val command = rootPath.resolve("fuzzyc2cpg.sh")
+  lazy val command: Path = rootPath.resolve("fuzzyc2cpg.sh")
 
   /**
     * Generate a CPG for the given input path.
diff --git a/console/src/main/scala/io/joern/console/cpgcreation/ImportCode.scala b/console/src/main/scala/io/joern/console/cpgcreation/ImportCode.scala
index 8973b272225e..0912fdca09ba 100644
--- a/console/src/main/scala/io/joern/console/cpgcreation/ImportCode.scala
+++ b/console/src/main/scala/io/joern/console/cpgcreation/ImportCode.scala
@@ -82,9 +82,10 @@ class ImportCode[T <: Project](console: io.joern.console.Console[T]) {
                             description: String = "Fuzzy Parser for C/C++",
                             extension: String = "c")
       extends Frontend(name, language, description) {
-    def fromString(str: String): Cpg = {
+
+    def fromString(str: String, args: List[String] = List()): Cpg = {
       withCodeInTmpFile(str, "tmp." + extension) { dir =>
-        apply(dir.path.toString)
+        apply(dir.path.toString, args = args)
       } match {
         case Failure(exception) => throw new ConsoleException(s"unable to generate cpg from given String", exception)
         case Success(value)     => value
diff --git a/console/src/main/scala/io/joern/console/cpgcreation/JavaCpgGenerator.scala b/console/src/main/scala/io/joern/console/cpgcreation/JavaCpgGenerator.scala
index 51a548e4fef6..34ead2819d76 100644
--- a/console/src/main/scala/io/joern/console/cpgcreation/JavaCpgGenerator.scala
+++ b/console/src/main/scala/io/joern/console/cpgcreation/JavaCpgGenerator.scala
@@ -33,7 +33,7 @@ case class JavaCpgGenerator(config: FrontendConfig, rootPath: Path) extends CpgG
       println("found .apk ending - will first transform it to a jar using dex2jar.sh")
       val dex2jar = rootPath.resolve("dex2jar.sh").toString
       runShellCommand(dex2jar, Seq(inputPath)).flatMap { _ =>
-        val jarPath = s"${inputPath}.jar"
+        val jarPath = s"$inputPath.jar"
         generateCommercial(jarPath, outputPath, namespaces)
       }
     } else {


Test patch:
diff --git a/console/src/test/scala/io/joern/console/ConsoleTests.scala b/console/src/test/scala/io/joern/console/ConsoleTests.scala
index e933bba40d64..37030565ad8d 100644
--- a/console/src/test/scala/io/joern/console/ConsoleTests.scala
+++ b/console/src/test/scala/io/joern/console/ConsoleTests.scala
@@ -19,7 +19,12 @@ class ConsoleTests extends AnyWordSpec with Matchers {
       console.importCode.toString.contains("| C") shouldBe true
     }
 
-    "allow importing code with specific module" in ConsoleFixture() { (console, codeDir) =>
+    "allow importing code with specific module (fuzzyc2cpg)" in ConsoleFixture() { (console, codeDir) =>
+      console.importCode.oldc(codeDir.toString)
+      console.workspace.numberOfProjects shouldBe 1
+    }
+
+    "allow importing code with specific module (c2cpg)" in ConsoleFixture() { (console, codeDir) =>
       console.importCode.c(codeDir.toString)
       console.workspace.numberOfProjects shouldBe 1
     }
@@ -27,7 +32,7 @@ class ConsoleTests extends AnyWordSpec with Matchers {
     "allow importing code" in ConsoleFixture() { (console, codeDir) =>
       console.importCode(codeDir.toString)
       console.workspace.numberOfProjects shouldBe 1
-      Set("main", "bar").subsetOf(console.cpg.method.name.toSet)
+      Set("main", "bar").subsetOf(console.cpg.method.name.toSet) shouldBe true
       console.project.appliedOverlays shouldBe List(Base.overlayName,
                                                     ControlFlow.overlayName,
                                                     TypeRelations.overlayName,
@@ -38,6 +43,41 @@ class ConsoleTests extends AnyWordSpec with Matchers {
                                                            CallGraph.overlayName)
     }
 
+    "allow importing code from file with defines and additional args" in ConsoleFixture() { (console, codeDir) =>
+      val code =
+        """
+          |#ifdef D
+          |int foo() {};
+          |#endif
+          |""".stripMargin
+      File.usingTemporaryFile("console", suffix = ".c", parent = Some(codeDir)) { file =>
+        file.write(code)
+        console.importCode.c(inputPath = codeDir.toString)
+        // importing without args should not yield foo
+        Set("foo").subsetOf(console.cpg.method.name.toSet) shouldBe false
+
+        // importing with args should yield foo
+        console.importCode.c(inputPath = codeDir.toString(), args = List("--define", "D"))
+        Set("foo").subsetOf(console.cpg.method.name.toSet) shouldBe true
+      }
+    }
+
+    "allow importing code from string with defines and additional args" in ConsoleFixture() { (console, _) =>
+      val code =
+        """
+          |#ifdef D
+          |int foo() {};
+          |#endif
+          |""".stripMargin
+      // importing without args should not yield foo
+      console.importCode.c.fromString(code)
+      Set("foo").subsetOf(console.cpg.method.name.toSet) shouldBe false
+
+      // importing with args should yield foo
+      console.importCode.c.fromString(code, List("--define", "D"))
+      Set("foo").subsetOf(console.cpg.method.name.toSet) shouldBe true
+    }
+
     "allow importing code and setting project name" in ConsoleFixture() { (console, codeDir) =>
       val projectName = "test"
       console.importCode(codeDir.toString, projectName)
@@ -71,7 +111,7 @@ class ConsoleTests extends AnyWordSpec with Matchers {
 
     "set project to active" in ConsoleFixture() { (console, codeDir) =>
       console.importCode(codeDir.toString, "foo")
-      Set("main", "bar").subsetOf(console.cpg.method.name.toSet)
+      Set("main", "bar").subsetOf(console.cpg.method.name.toSet) shouldBe true
     }
 
   }
@@ -114,7 +154,7 @@ class ConsoleTests extends AnyWordSpec with Matchers {
       try {
         console.importCpg(tmpCpg.toString)
         console.workspace.numberOfProjects shouldBe 1
-        Set("main", "bar").subsetOf(console.cpg.method.name.toSet)
+        Set("main", "bar").subsetOf(console.cpg.method.name.toSet) shouldBe true
         console.project.appliedOverlays shouldBe List(Base.overlayName,
                                                       ControlFlow.overlayName,
                                                       TypeRelations.overlayName,
@@ -130,7 +170,7 @@ class ConsoleTests extends AnyWordSpec with Matchers {
         console.importCpg(tmpCpg.toString, "foobar")
         console.workspace.numberOfProjects shouldBe 1
         console.workspace.project("foobar") should not be empty
-        Set("main", "bar").subsetOf(console.cpg.method.name.toSet)
+        Set("main", "bar").subsetOf(console.cpg.method.name.toSet) shouldBe true
       } finally {
         Some(tmpCpg).find(_.exists).foreach(_.delete())
       }
@@ -166,7 +206,7 @@ class ConsoleTests extends AnyWordSpec with Matchers {
         console.importCpg(tmpCpg.toString)
         console.importCpg(tmpCpg.toString)
         console.workspace.numberOfProjects shouldBe 1
-        Set("main", "bar").subsetOf(console.cpg.method.name.toSet)
+        Set("main", "bar").subsetOf(console.cpg.method.name.toSet) shouldBe true
         console.project.appliedOverlays shouldBe List(Base.overlayName,
                                                       ControlFlow.overlayName,
                                                       TypeRelations.overlayName,
@@ -210,9 +250,9 @@ class ConsoleTests extends AnyWordSpec with Matchers {
       console.importCode(codeDir.toString, "foo")
       console.importCode(codeDir.toString, "bar")
       console.close
-      a[RuntimeException] should be thrownBy (console.cpg)
+      a[RuntimeException] should be thrownBy console.cpg
       console.open("foo")
-      Set("main", "bar").subsetOf(console.cpg.method.name.toSet)
+      Set("main", "bar").subsetOf(console.cpg.method.name.toSet) shouldBe true
     }
   }
 
@@ -240,14 +280,14 @@ class ConsoleTests extends AnyWordSpec with Matchers {
                                                     ControlFlow.overlayName,
                                                     TypeRelations.overlayName,
                                                     CallGraph.overlayName)
-      val numOverlayFilesBefore = console.project.path.resolve("overlays").toFile.list().size
+      val numOverlayFilesBefore = console.project.path.resolve("overlays").toFile.list().length
       numOverlayFilesBefore shouldBe 4
       console._runAnalyzer(new Base, new ControlFlow, new TypeRelations, new CallGraph)
       console.project.appliedOverlays shouldBe List(Base.overlayName,
                                                     ControlFlow.overlayName,
                                                     TypeRelations.overlayName,
                                                     CallGraph.overlayName)
-      console.project.path.resolve("overlays").toFile.list().size shouldBe numOverlayFilesBefore
+      console.project.path.resolve("overlays").toFile.list().length shouldBe numOverlayFilesBefore
     }
 
     "store directory of zip files for each overlay in project" in ConsoleFixture() { (console, codeDir) =>
@@ -381,7 +421,7 @@ class ConsoleTests extends AnyWordSpec with Matchers {
   "switchWorkspace" should {
 
     "create workspace if directory does not exist" in ConsoleFixture() { (console, _) =>
-      val otherWorkspaceDir = ("/tmp" / "workspace-doesNotExist")
+      val otherWorkspaceDir = "/tmp" / "workspace-doesNotExist"
       try {
         otherWorkspaceDir.exists shouldBe false
         console.switchWorkspace(otherWorkspaceDir.path.toString)
diff --git a/console/src/test/scala/io/joern/console/testing/ConsoleFixture.scala b/console/src/test/scala/io/joern/console/testing/ConsoleFixture.scala
index 20fc9f888396..e8d8d44c9216 100644
--- a/console/src/test/scala/io/joern/console/testing/ConsoleFixture.scala
+++ b/console/src/test/scala/io/joern/console/testing/ConsoleFixture.scala
@@ -2,10 +2,12 @@ package io.joern.console.testing
 
 import better.files.Dsl.mkdir
 import better.files.File
+import io.joern.c2cpg.C2Cpg
 import io.joern.console.cpgcreation.{CpgGenerator, CpgGeneratorFactory, ImportCode}
 import io.joern.console.workspacehandling.{Project, ProjectFile, WorkspaceLoader}
 import io.joern.console.{Console, ConsoleConfig, DefaultAmmoniteExecutor, FrontendConfig, InstallConfig}
 import io.joern.fuzzyc2cpg.FuzzyC2Cpg
+import io.shiftleft.codepropertygraph.generated.Languages
 
 import java.nio.file.Path
 
@@ -41,13 +43,23 @@ class TestConsole(workspaceDir: String)
   override def importCode: ImportCode[Project] = new ImportCode(this) {
     override val generatorFactory = new TestCpgGeneratorFactory(config)
 
-    override def c: SourceBasedFrontend = new SourceBasedFrontend("testCFrontend") {
+    override def oldc: SourceBasedFrontend = new SourceBasedFrontend("testFuzzyCFrontend", language = Languages.C) {
       override def cpgGeneratorForLanguage(language: String,
                                            config: FrontendConfig,
                                            rootPath: Path,
                                            args: List[String]): Option[CpgGenerator] =
         generatorFactory.forLanguage(language)
     }
+
+    override def c: SourceBasedFrontend = new SourceBasedFrontend("testCFrontend", language = Languages.NEWC) {
+      override def cpgGeneratorForLanguage(language: String,
+                                           config: FrontendConfig,
+                                           rootPath: Path,
+                                           args: List[String]): Option[CpgGenerator] = {
+        val newConfig = new ConsoleConfig(TestConsole.this.config.install, config.withArgs(args))
+        new TestCpgGeneratorFactory(newConfig).forLanguage(language)
+      }
+    }
   }
 }
 
@@ -58,8 +70,30 @@ class TestCpgGeneratorFactory(config: ConsoleConfig) extends CpgGeneratorFactory
     Some(new FuzzyCTestingFrontend)
   }
 
-  override def forLanguage(language: String): Option[CpgGenerator] = {
-    Some(new FuzzyCTestingFrontend)
+  override def forLanguage(language: String): Option[CpgGenerator] = language match {
+    case Languages.C    => Some(new FuzzyCTestingFrontend)
+    case Languages.NEWC => Some(new CTestingFrontend)
+    case _              => None // no other languages are tested here
+  }
+
+  private class CTestingFrontend extends CpgGenerator {
+
+    override def generate(inputPath: String, outputPath: String, namespaces: List[String]): Option[String] = {
+      val c2cpg = new C2Cpg()
+      val defines = config.frontend.cmdLineParams.toList
+        .sliding(2)
+        .toList
+        .collect {
+          case List(h, t) if h == "--define" => t
+        }
+      val cpg =
+        c2cpg.runAndOutput(C2Cpg.Config(Set(inputPath), outputPath, defines = defines.toSet))
+      cpg.close()
+      Some(outputPath)
+    }
+
+    def isAvailable: Boolean = true
+
   }
 
   private class FuzzyCTestingFrontend extends CpgGenerator {