Repository: joernio/joern (scala)

Issue:
[C#] Do not summarize dependency if already summarized
Currently we override any `builtins` summarization if we encounter the same package in a `.csproj` file through download dependencies. We want the builtins ( or already present dependencies ) to take precedence.

Pull request description:
* Adds a check to existing summaries if existing namespacing starting with the dependency name exists.
* Added `TypeLike` addition handler to handle member collisions.
* Upgraded DotNetAstGen version

Resolves #4368

Patch:
diff --git a/joern-cli/frontends/csharpsrc2cpg/src/main/resources/application.conf b/joern-cli/frontends/csharpsrc2cpg/src/main/resources/application.conf
index e202f14be487..008f531d4a7e 100644
--- a/joern-cli/frontends/csharpsrc2cpg/src/main/resources/application.conf
+++ b/joern-cli/frontends/csharpsrc2cpg/src/main/resources/application.conf
@@ -1,3 +1,3 @@
 csharpsrc2cpg {
-    dotnetastgen_version: "0.26.0"
+    dotnetastgen_version: "0.27.0"
 }
diff --git a/joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/datastructures/CSharpProgramSummary.scala b/joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/datastructures/CSharpProgramSummary.scala
index 248f047a3c7f..a07a7c07063a 100644
--- a/joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/datastructures/CSharpProgramSummary.scala
+++ b/joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/datastructures/CSharpProgramSummary.scala
@@ -119,4 +119,10 @@ case class CSharpMethod(name: String, returnType: String, parameterTypes: List[(
     extends MethodLike derives ReadWriter
 
 case class CSharpType(name: String, methods: List[CSharpMethod], fields: List[CSharpField])
-    extends TypeLike[CSharpMethod, CSharpField] derives ReadWriter
+    extends TypeLike[CSharpMethod, CSharpField] derives ReadWriter {
+  @targetName("add")
+  override def +(o: TypeLike[CSharpMethod, CSharpField]): TypeLike[CSharpMethod, CSharpField] = {
+    this.copy(methods = mergeMethods(o), fields = mergeFields(o))
+  }
+
+}
diff --git a/joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/utils/DependencyDownloader.scala b/joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/utils/DependencyDownloader.scala
index 51a735304cd3..86cb090a980d 100644
--- a/joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/utils/DependencyDownloader.scala
+++ b/joern-cli/frontends/csharpsrc2cpg/src/main/scala/io/joern/csharpsrc2cpg/utils/DependencyDownloader.scala
@@ -54,8 +54,10 @@ class DependencyDownloader(
     *   true if the dependency is already in the given summary, false if otherwise.
     */
   private def isAlreadySummarized(dependency: Dependency): Boolean = {
-    // TODO: Check internalSummaries too
-    internalPackages.contains(dependency.name)
+    // TODO: `namespace` != `packageId`, so we may want to have summaries store the `packageId` in future
+    internalProgramSummary.namespaceToType.keySet.exists(_.startsWith(dependency.name)) || internalPackages.contains(
+      dependency.name
+    )
   }
 
   private case class NuGetPackageVersions(versions: List[String]) derives ReadWriter
diff --git a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/datastructures/RubyProgramSummary.scala b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/datastructures/RubyProgramSummary.scala
index 156c1f6c7dbc..5593a6211b6a 100644
--- a/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/datastructures/RubyProgramSummary.scala
+++ b/joern-cli/frontends/rubysrc2cpg/src/main/scala/io/joern/rubysrc2cpg/datastructures/RubyProgramSummary.scala
@@ -1,7 +1,8 @@
 package io.joern.rubysrc2cpg.datastructures
 
-import io.joern.x2cpg.datastructures.{FieldLike, MethodLike, ProgramSummary, TypeLike}
 import io.joern.x2cpg.Defines as XDefines
+import io.joern.x2cpg.datastructures.{FieldLike, MethodLike, ProgramSummary, TypeLike}
+
 import scala.annotation.targetName
 
 class RubyProgramSummary(
@@ -33,6 +34,12 @@ case class RubyField(name: String, typeName: String) extends FieldLike
 
 case class RubyType(name: String, methods: List[RubyMethod], fields: List[RubyField])
     extends TypeLike[RubyMethod, RubyField] {
+
+  @targetName("add")
+  override def +(o: TypeLike[RubyMethod, RubyField]): TypeLike[RubyMethod, RubyField] = {
+    this.copy(methods = mergeMethods(o), fields = mergeFields(o))
+  }
+
   def hasConstructor: Boolean = {
     methods.exists(_.name == XDefines.ConstructorMethodName)
   }
diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/datastructures/ProgramSummary.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/datastructures/ProgramSummary.scala
index eeb2afe6a22f..07964e399100 100644
--- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/datastructures/ProgramSummary.scala
+++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/datastructures/ProgramSummary.scala
@@ -2,6 +2,7 @@ package io.joern.x2cpg.datastructures
 
 import io.shiftleft.codepropertygraph.generated.nodes.DeclarationNew
 
+import scala.annotation.targetName
 import scala.collection.immutable.Map
 import scala.collection.mutable
 import scala.reflect.ClassTag
@@ -43,13 +44,21 @@ object ProgramSummary {
 
   /** Combines two namespace-to-type maps.
     */
-  def combine[T <: TypeLike[_, _]](a: Map[String, Set[T]], b: Map[String, Set[T]]): Map[String, Set[T]] = {
+  def combine[T <: TypeLike[M, F], M <: MethodLike, F <: FieldLike](
+    a: Map[String, Set[T]],
+    b: Map[String, Set[T]]
+  ): Map[String, Set[T]] = {
     val accumulator = mutable.HashMap.from(a)
 
-    b.keySet.foreach(k =>
-      accumulator.updateWith(k) {
-        case Some(existing) => Option(a.getOrElse(k, Set.empty) ++ b.getOrElse(k, Set.empty) ++ existing)
-        case None           => Option(a.getOrElse(k, Set.empty) ++ b.getOrElse(k, Set.empty))
+    b.keySet.foreach(namespace =>
+      accumulator.updateWith(namespace) {
+        case Some(existing) =>
+          val types = (existing.toList ++ b(namespace).toList)
+            .groupBy(_.name)
+            .map { case (_, ts) => ts.reduce((a, b) => (a + b).asInstanceOf[T]) }
+            .toSet
+          Option(types)
+        case None => b.get(namespace)
       }
     )
     accumulator.toMap
@@ -299,6 +308,38 @@ trait TypeLike[M <: MethodLike, F <: FieldLike] {
     */
   def fields: List[F]
 
+  /** Adds the contents of the two types and produces a new type.
+    * @param o
+    *   the other type-like.
+    * @return
+    *   a type-like that is the combination of the two, with precedence to colliding contents to this type (LHS).
+    */
+  @targetName("add")
+  def +(o: TypeLike[M, F]): TypeLike[M, F]
+
+  /** Helper method for creating the sum of two type-like's methods, while preferring this types' methods on collisions.
+    * @param o
+    *   the other type-like.
+    * @return
+    *   the combination of the two type-like's methods.
+    */
+  protected def mergeMethods(o: TypeLike[M, _]): List[M] = {
+    val methodNames = methods.map(_.name).toSet
+    methods ++ o.methods.filterNot(m => methodNames.contains(m.name))
+  }
+
+  /** Helper method for creating the sum of two type-like's fields, while preferring this types' fields on collisions.
+    *
+    * @param o
+    *   the other type-like.
+    * @return
+    *   the combination of the two type-like's fields.
+    */
+  protected def mergeFields(o: TypeLike[_, F]): List[F] = {
+    val fieldNames = fields.map(_.name).toSet
+    fields ++ o.fields.filterNot(f => fieldNames.contains(f.name))
+  }
+
 }
 
 /** An entity that is a member to some type or module.


Test patch:
diff --git a/joern-cli/frontends/csharpsrc2cpg/src/test/scala/io/joern/csharpsrc2cpg/io/DependencyDownloadTests.scala b/joern-cli/frontends/csharpsrc2cpg/src/test/scala/io/joern/csharpsrc2cpg/io/DependencyDownloadTests.scala
index 5f2cf89257a0..a00719f6f4ce 100644
--- a/joern-cli/frontends/csharpsrc2cpg/src/test/scala/io/joern/csharpsrc2cpg/io/DependencyDownloadTests.scala
+++ b/joern-cli/frontends/csharpsrc2cpg/src/test/scala/io/joern/csharpsrc2cpg/io/DependencyDownloadTests.scala
@@ -42,8 +42,8 @@ class DependencyDownloadTests extends CSharpCode2CpgFixture {
       summary.typesUnderNamespace("CSharpx") should not be empty
     }
 
-    "not summarize NewtonsoftJson (as it publishes without PDB files)" in {
-      summary.typesUnderNamespace("Microsoft.AspNetCore.Mvc.NewtonsoftJson") shouldBe empty
+    "manage to summarize NewtonsoftJson (as it publishes without PDB files, but DotNetAstGen can generate these)" in {
+      summary.typesUnderNamespace("Microsoft.AspNetCore.Mvc.NewtonsoftJson") should not be empty
     }
 
   }
diff --git a/joern-cli/frontends/csharpsrc2cpg/src/test/scala/io/joern/csharpsrc2cpg/querying/ast/DependencyTests.scala b/joern-cli/frontends/csharpsrc2cpg/src/test/scala/io/joern/csharpsrc2cpg/querying/ast/DependencyTests.scala
index c07396f9dedc..0b9ae832c9b5 100644
--- a/joern-cli/frontends/csharpsrc2cpg/src/test/scala/io/joern/csharpsrc2cpg/querying/ast/DependencyTests.scala
+++ b/joern-cli/frontends/csharpsrc2cpg/src/test/scala/io/joern/csharpsrc2cpg/querying/ast/DependencyTests.scala
@@ -66,4 +66,42 @@ class DependencyTests extends CSharpCode2CpgFixture {
 
   }
 
+  "a `csproj` file specifying a builtin dependency" should {
+
+    val cpg = code("""
+        |using System;
+        |using Microsoft.EntityFrameworkCore;
+        |
+        |public class Foo {
+        |
+        | static void bar(ModelBuilder modelBuilder)
+        | {
+        |   modelBuilder.Entity("test");
+        | }
+        |
+        |}
+        |""".stripMargin)
+      .moreCode(
+        """
+        |<Project Sdk="Microsoft.NET.Sdk">
+        |    <ItemGroup>
+        |      <PackageReference Include="Microsoft.EntityFrameworkCore" Version="doesNotExist" />
+        |    </ItemGroup>
+        |</Project>
+        |""".stripMargin,
+        "Foo.csproj"
+      )
+      .withConfig(Config().withDownloadDependencies(true))
+
+    "resolve the call from the local builtins, as the specified dependency would not successfully download" in {
+      inside(cpg.call("Entity").headOption) {
+        case Some(entity) =>
+          entity.methodFullName shouldBe "Microsoft.EntityFrameworkCore.ModelBuilder.Entity:Microsoft.EntityFrameworkCore.Metadata.Builders.EntityTypeBuilder(System.String)"
+        case None =>
+          fail("Expected a call node for `Entity`")
+      }
+    }
+
+  }
+
 }
diff --git a/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/datastructures/ProgramSummaryTests.scala b/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/datastructures/ProgramSummaryTests.scala
index 9f6e2b088e43..4da074066303 100644
--- a/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/datastructures/ProgramSummaryTests.scala
+++ b/joern-cli/frontends/x2cpg/src/test/scala/io/joern/x2cpg/datastructures/ProgramSummaryTests.scala
@@ -3,9 +3,12 @@ package io.joern.x2cpg.datastructures
 import org.scalatest.matchers.should.Matchers
 import org.scalatest.wordspec.AnyWordSpec
 import io.shiftleft.codepropertygraph.generated.nodes.DeclarationNew
+
 import scala.collection.mutable
 import org.scalatest.Inside
 
+import scala.annotation.targetName
+
 class ProgramSummaryTests extends AnyWordSpec with Matchers with Inside {
 
   "a typed program summary based off of Java" should {
@@ -113,6 +116,11 @@ class ProgramSummaryTests extends AnyWordSpec with Matchers with Inside {
 
   case class Field(name: String, typeName: String) extends FieldLike
 
-  case class Typ(name: String, methods: List[Method], fields: List[Field]) extends TypeLike[Method, Field]
+  case class Typ(name: String, methods: List[Method], fields: List[Field]) extends TypeLike[Method, Field] {
+    @targetName("add")
+    override def +(o: TypeLike[Method, Field]): TypeLike[Method, Field] = {
+      this.copy(methods = mergeMethods(o), fields = mergeFields(o))
+    }
+  }
 
 }