Repository: joernio/joern
Pull request #1766 (2022-09-20 08:13:43)

Replace deprecated usage of CpgPass and DiffGraph.newBuilder


Patch:
diff --git a/console/src/main/scala/io/joern/console/Commit.scala b/console/src/main/scala/io/joern/console/Commit.scala
index 5128c8b58..aa2fd4c7d 100644
--- a/console/src/main/scala/io/joern/console/Commit.scala
+++ b/console/src/main/scala/io/joern/console/Commit.scala
@@ -1,15 +1,17 @@
 package io.joern.console
 
-import io.shiftleft.passes.{CpgPass, DiffGraph}
+import io.shiftleft.passes.CpgPassBase
+import io.shiftleft.passes.SimpleCpgPass
 import io.shiftleft.semanticcpg.layers.{LayerCreator, LayerCreatorContext, LayerCreatorOptions}
+import overflowdb.BatchedUpdate.DiffGraphBuilder
 
 object Commit {
   val overlayName: String = "commit"
   val description: String = "Apply current custom diffgraph"
-  def defaultOpts         = new CommitOptions(DiffGraph.newBuilder)
+  def defaultOpts         = new CommitOptions(new DiffGraphBuilder)
 }
 
-class CommitOptions(var diffGraphBuilder: DiffGraph.Builder) extends LayerCreatorOptions
+class CommitOptions(var diffGraphBuilder: DiffGraphBuilder) extends LayerCreatorOptions
 
 class Commit(opts: CommitOptions) extends LayerCreator {
 
@@ -18,12 +20,14 @@ class Commit(opts: CommitOptions) extends LayerCreator {
   override val storeOverlayName: Boolean = false
 
   override def create(context: LayerCreatorContext, storeUndoInfo: Boolean): Unit = {
-    val pass: CpgPass = new CpgPass(context.cpg) {
-      override val name                       = "commit"
-      override def run(): Iterator[DiffGraph] = Iterator(opts.diffGraphBuilder.build())
+    val pass: CpgPassBase = new SimpleCpgPass(context.cpg) {
+      override val name = "commit"
+      override def run(diffGraphBuilder: DiffGraphBuilder): Unit = {
+        diffGraphBuilder.absorb(opts.diffGraphBuilder)
+      }
     }
     runPass(pass, context, storeUndoInfo)
-    opts.diffGraphBuilder = DiffGraph.newBuilder
+    opts.diffGraphBuilder = new DiffGraphBuilder
   }
 
 }
diff --git a/console/src/main/scala/io/joern/console/Run.scala b/console/src/main/scala/io/joern/console/Run.scala
index cd526da0a..7c1fceffe 100644
--- a/console/src/main/scala/io/joern/console/Run.scala
+++ b/console/src/main/scala/io/joern/console/Run.scala
@@ -1,6 +1,7 @@
 package io.joern.console
 
-import io.shiftleft.passes.{CpgPass, DiffGraph}
+import io.shiftleft.passes.CpgPassBase
+import io.shiftleft.passes.SimpleCpgPass
 import io.shiftleft.semanticcpg.language.HasStoreMethod
 import io.shiftleft.semanticcpg.layers.{LayerCreator, LayerCreatorContext}
 import org.reflections8.Reflections
@@ -16,12 +17,10 @@ object Run {
       override val description: String = "A custom pass"
 
       override def create(context: LayerCreatorContext, storeUndoInfo: Boolean): Unit = {
-        val pass: CpgPass = new CpgPass(console.cpg) {
+        val pass: CpgPassBase = new SimpleCpgPass(console.cpg) {
           override val name = "custom"
-          override def run(): Iterator[DiffGraph] = {
-            implicit val diffGraph: DiffGraph.Builder = DiffGraph.newBuilder
-            query.store()
-            Iterator(diffGraph.build())
+          override def run(diffGraph: DiffGraphBuilder): Unit = {
+            query.store()(diffGraph)
           }
         }
         runPass(pass, context, storeUndoInfo)
@@ -66,8 +65,8 @@ object Run {
          |
          |val opts = new OptsDynamic()
          |
-         | import io.shiftleft.passes.DiffGraph
-         | implicit def _diffGraph: DiffGraph.Builder = opts.commit.diffGraphBuilder
+         | import overflowdb.BatchedUpdate.DiffGraphBuilder
+         | implicit def _diffGraph: DiffGraphBuilder = opts.commit.diffGraphBuilder
          | def diffGraph = _diffGraph
          |""".stripMargin
 
diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/NewNodeSteps.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/NewNodeSteps.scala
index f0ec07570..7446524b2 100644
--- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/NewNodeSteps.scala
+++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/NewNodeSteps.scala
@@ -1,19 +1,19 @@
 package io.shiftleft.semanticcpg.language
 
 import io.shiftleft.codepropertygraph.generated.nodes.NewNode
-import io.shiftleft.passes.DiffGraph
 import overflowdb.traversal._
+import overflowdb.BatchedUpdate.DiffGraphBuilder
 
 trait HasStoreMethod {
-  def store()(implicit diffBuilder: DiffGraph.Builder): Unit
+  def store()(implicit diffBuilder: DiffGraphBuilder): Unit
 }
 
 class NewNodeSteps[A <: NewNode](val traversal: Traversal[A]) extends HasStoreMethod {
 
-  override def store()(implicit diffBuilder: DiffGraph.Builder): Unit =
+  override def store()(implicit diffBuilder: DiffGraphBuilder): Unit =
     traversal.sideEffect(storeRecursively).iterate()
 
-  private def storeRecursively(newNode: NewNode)(implicit diffBuilder: DiffGraph.Builder): Unit = {
+  private def storeRecursively(newNode: NewNode)(implicit diffBuilder: DiffGraphBuilder): Unit = {
     diffBuilder.addNode(newNode)
   }
 
diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/NewTagNodePairTraversal.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/NewTagNodePairTraversal.scala
index 280bdece1..a405d8a1c 100644
--- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/NewTagNodePairTraversal.scala
+++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/NewTagNodePairTraversal.scala
@@ -1,20 +1,21 @@
 package io.shiftleft.semanticcpg.language
 
 import io.shiftleft.codepropertygraph.generated.EdgeTypes
-import io.shiftleft.codepropertygraph.generated.nodes.{NewNode, NewTagNodePair, StoredNode}
-import io.shiftleft.passes.DiffGraph
+import io.shiftleft.codepropertygraph.generated.nodes.{NewNode, NewTagNodePair}
+import io.shiftleft.codepropertygraph.generated.nodes.StoredNode
 import overflowdb.traversal._
+import overflowdb.BatchedUpdate.DiffGraphBuilder
 
 class NewTagNodePairTraversal(traversal: Traversal[NewTagNodePair]) extends HasStoreMethod {
 
-  override def store()(implicit diffGraph: DiffGraph.Builder): Unit = {
+  override def store()(implicit diffGraph: DiffGraphBuilder): Unit = {
     traversal.foreach { tagNodePair =>
       val tag      = tagNodePair.tag
       val tagValue = tagNodePair.node
       diffGraph.addNode(tag.asInstanceOf[NewNode])
       tagValue match {
         case tagValue: StoredNode =>
-          diffGraph.addEdgeFromOriginal(tagValue, tag.asInstanceOf[NewNode], EdgeTypes.TAGGED_BY)
+          diffGraph.addEdge(tagValue, tag.asInstanceOf[NewNode], EdgeTypes.TAGGED_BY)
         case tagValue: NewNode =>
           diffGraph.addEdge(tagValue, tag.asInstanceOf[NewNode], EdgeTypes.TAGGED_BY, Nil)
       }


Test patch:
diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/testing/package.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/testing/package.scala
index 66e22db19..a659e5d66 100644
--- a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/testing/package.scala
+++ b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/testing/package.scala
@@ -3,8 +3,9 @@ package io.shiftleft.semanticcpg
 import io.shiftleft.codepropertygraph.Cpg
 import io.shiftleft.codepropertygraph.generated.nodes._
 import io.shiftleft.codepropertygraph.generated.{EdgeTypes, Languages, ModifierTypes}
-import io.shiftleft.passes.{CpgPass, DiffGraph}
+import io.shiftleft.passes.SimpleCpgPass
 import io.shiftleft.semanticcpg.language._
+import overflowdb.BatchedUpdate.DiffGraphBuilder
 
 package object testing {
 
@@ -12,7 +13,7 @@ package object testing {
 
     def apply(): MockCpg = new MockCpg
 
-    def apply(f: (DiffGraph.Builder, Cpg) => Unit): MockCpg = new MockCpg().withCustom(f)
+    def apply(f: (DiffGraphBuilder, Cpg) => Unit): MockCpg = new MockCpg().withCustom(f)
 
   }
 
@@ -125,7 +126,7 @@ package object testing {
       paramTags: List[(String, String)] = List()
     ): MockCpg =
       withCustom { (graph, cpg) =>
-        implicit val diffGraph: DiffGraph.Builder = graph
+        implicit val diffGraph: DiffGraphBuilder = graph
         methodTags.foreach { case (k, v) =>
           cpg.method.name(methodName).newTagNodePair(k, v).store()
         }
@@ -198,12 +199,12 @@ package object testing {
       graph.addNode(newNode)
     }
 
-    def withCustom(f: (DiffGraph.Builder, Cpg) => Unit): MockCpg = {
-      val diffGraph = new DiffGraph.Builder
+    def withCustom(f: (DiffGraphBuilder, Cpg) => Unit): MockCpg = {
+      val diffGraph = new DiffGraphBuilder
       f(diffGraph, cpg)
-      class MyPass extends CpgPass(cpg) {
-        override def run(): Iterator[DiffGraph] = {
-          Iterator(diffGraph.build())
+      class MyPass extends SimpleCpgPass(cpg) {
+        override def run(diffBuilder: DiffGraphBuilder): Unit = {
+          diffBuilder.absorb(diffGraph)
         }
       }
       new MyPass().createAndApply()
diff --git a/semanticcpg/src/test/scala/io/shiftleft/semanticcpg/language/NewNodeStepsTests.scala b/semanticcpg/src/test/scala/io/shiftleft/semanticcpg/language/NewNodeStepsTests.scala
index c882fc460..5d5889059 100644
--- a/semanticcpg/src/test/scala/io/shiftleft/semanticcpg/language/NewNodeStepsTests.scala
+++ b/semanticcpg/src/test/scala/io/shiftleft/semanticcpg/language/NewNodeStepsTests.scala
@@ -2,22 +2,22 @@ package io.shiftleft.semanticcpg.language
 
 import io.shiftleft.codepropertygraph.Cpg
 import io.shiftleft.codepropertygraph.generated.nodes._
-import io.shiftleft.codepropertygraph.generated.{ModifierTypes, PropertyNames}
-import io.shiftleft.passes.DiffGraph
 import org.scalatest.matchers.should.Matchers
 import org.scalatest.wordspec.AnyWordSpec
+import overflowdb.BatchedUpdate
+import overflowdb.BatchedUpdate.DiffGraphBuilder
 
 import scala.jdk.CollectionConverters._
 
-class NewNodeStepsTest extends AnyWordSpec with Matchers {
-  import NewNodeNodeStepsTest._
+class NewNodeStepsTests extends AnyWordSpec with Matchers {
+  import NewNodeStepsTests._
 
   "stores NewNodes" in {
-    implicit val diffGraphBuilder = DiffGraph.newBuilder
-    val newNode                   = newTestNode()
+    implicit val diffGraphBuilder: DiffGraphBuilder = new DiffGraphBuilder
+    val newNode                                     = newTestNode()
     new NewNodeSteps(newNode.start).store()
     val diffGraph = diffGraphBuilder.build()
-    diffGraph.nodes.toList shouldBe List(newNode)
+    diffGraph.changes.toList shouldBe List(newNode)
   }
 
   "can access the node label" in {
@@ -28,42 +28,38 @@ class NewNodeStepsTest extends AnyWordSpec with Matchers {
   "stores containedNodes and connecting edge" when {
 
     "embedding a StoredNode and a NewNode" in {
-      implicit val diffGraphBuilder = DiffGraph.newBuilder
-      val cpg                       = Cpg.emptyCpg
-      val existingContainedNode     = cpg.graph.addNode(42L, "MODIFIER").asInstanceOf[StoredNode]
-      existingContainedNode.setProperty(PropertyNames.MODIFIER_TYPE, ModifierTypes.NATIVE)
+      implicit val diffGraphBuilder: DiffGraphBuilder = new DiffGraphBuilder
+      val cpg                                         = Cpg.emptyCpg
+      val existingContainedNode                       = cpg.graph.addNode(42L, "MODIFIER").asInstanceOf[StoredNode]
       cpg.graph.V().asScala.toSet shouldBe Set(existingContainedNode)
 
       val newContainedNode = newTestNode()
       val newNode          = newTestNode(evidence = List(existingContainedNode, newContainedNode))
       new NewNodeSteps(newNode.start).store()
       val diffGraph = diffGraphBuilder.build()
-      diffGraph.nodes.toSet shouldBe Set(newNode)
-      diffGraph.edges shouldBe Nil
+      diffGraph.changes.toSet shouldBe Set(newNode)
 
-      DiffGraph.Applier.applyDiff(diffGraph, cpg, undoable = false, keyPool = None)
+      BatchedUpdate.applyDiff(cpg.graph, diffGraph)
       cpg.graph.V().asScala.length shouldBe 3
     }
 
     "embedding a NewNode recursively" in {
-      implicit val diffGraphBuilder = DiffGraph.newBuilder
-      val cpg                       = Cpg.emptyCpg
-      val newContainedNodeL1        = newTestNode()
-      val newContainedNodeL0        = newTestNode(evidence = List(newContainedNodeL1))
-      val newNode                   = newTestNode(evidence = List(newContainedNodeL0))
+      implicit val diffGraphBuilder: DiffGraphBuilder = new DiffGraphBuilder
+      val cpg                                         = Cpg.emptyCpg
+      val newContainedNodeL1                          = newTestNode()
+      val newContainedNodeL0                          = newTestNode(evidence = List(newContainedNodeL1))
+      val newNode                                     = newTestNode(evidence = List(newContainedNodeL0))
       new NewNodeSteps(newNode.start).store()
       val diffGraph = diffGraphBuilder.build()
 
-      diffGraph.nodes.toSet shouldBe Set(newNode)
-      diffGraph.edges.toSet shouldBe Set.empty
-      DiffGraph.Applier.applyDiff(diffGraph, cpg, undoable = false, keyPool = None)
+      diffGraph.changes.toSet shouldBe Set(newNode)
+      BatchedUpdate.applyDiff(cpg.graph, diffGraph)
       cpg.graph.V().asScala.size shouldBe 3
     }
 
   }
 }
 
-object NewNodeNodeStepsTest {
-  def newTestNode(evidence: Seq[AbstractNode] = Seq.empty): NewFinding =
-    NewFinding().evidence(evidence)
+object NewNodeStepsTests {
+  def newTestNode(evidence: Seq[AbstractNode] = Seq.empty): NewFinding = NewFinding().evidence(evidence)
 }
diff --git a/semanticcpg/src/test/scala/io/shiftleft/semanticcpg/language/types/structure/MethodTests.scala b/semanticcpg/src/test/scala/io/shiftleft/semanticcpg/language/types/structure/MethodTests.scala
index a42454717..bca4a50ae 100644
--- a/semanticcpg/src/test/scala/io/shiftleft/semanticcpg/language/types/structure/MethodTests.scala
+++ b/semanticcpg/src/test/scala/io/shiftleft/semanticcpg/language/types/structure/MethodTests.scala
@@ -2,6 +2,7 @@ package io.shiftleft.semanticcpg.language.types.structure
 
 import io.shiftleft.codepropertygraph.generated.EdgeTypes
 import io.shiftleft.codepropertygraph.generated.nodes.{CfgNode, Expression, Literal, Method, NamespaceBlock, TypeDecl}
+import io.shiftleft.codepropertygraph.Cpg
 import io.shiftleft.semanticcpg.language._
 import io.shiftleft.semanticcpg.testing.MockCpg
 import org.scalatest.matchers.should.Matchers
@@ -9,10 +10,10 @@ import org.scalatest.wordspec.AnyWordSpec
 
 class MethodTests extends AnyWordSpec with Matchers {
 
-  val cpg = MockCpg()
+  private val cpg: Cpg = MockCpg()
     .withNamespace("namespace")
     .withTypeDecl("TypeDecl", inNamespace = Some("namespace"))
-    .withMethod("foo", inTypeDecl = Some("TypeDecl"), external = false)
+    .withMethod("foo", inTypeDecl = Some("TypeDecl"))
     .withMethod("bar", inTypeDecl = Some("TypeDecl"), external = true)
     .withCallInMethod("foo", "call")
     .withCallInMethod("foo", "call2")
@@ -56,7 +57,7 @@ class MethodTests extends AnyWordSpec with Matchers {
           .withCustom { case (graph, cpg) =>
             val namespaceBlock = cpg.namespaceBlock("namespace").head
             val method         = cpg.method("foo").head
-            graph.addNodeProperty(method, Method.PropertyNames.AstParentType, NamespaceBlock.Label)
+            graph.setNodeProperty(method, Method.PropertyNames.AstParentType, NamespaceBlock.Label)
             graph.addEdge(namespaceBlock, method, EdgeTypes.AST)
           }
           .cpg