Repository: joernio/joern (scala)

Issue:
[Feature] Python Built-in Function Full Names
**Is your feature request related to a problem? Please describe.**
Python CPG method full names for calls of built-in functions are "unknown" but if there is no shadowed function name that is in the current scope of the compilation unit, i.e., module, then we can safely assume that it is a built-in function and give it some kind of full name. For example, `jssrc2cpg` prefixes built-in types with `__ecma`, e.g. `__ecma.String`.

Having a field under `methodFullName` that is set will help understand which calls are not resolved, and thus we don't unnecessarily try to recover types for these methods.

**Describe the solution you'd like**
To detect built-in functions and/or types and prefix them accordingly. The built-in functions, constants, and types are [in these docs](https://docs.python.org/3/library/index.html). This seems like each built-in structure can be incrementally supported.

Right now, some fields declared under classes are prefixed with `__builtins__`. We can simply continue this.

**Describe alternatives you've considered**
The method full name could simply match the name.

**Additional context**
More standard types are here: https://en.wikipedia.org/wiki/Python_(programming_language)

Pull request description:
- Added entrypoint to base class for pre-populating
- Pre-populated symbol table with built-in function names
- Moved Python passes from `x2cpg` to `pysrc2cpg`

Resolves #2207

Patch:
diff --git a/console/build.sbt b/console/build.sbt
index b94df19192a4..b50313741fe4 100644
--- a/console/build.sbt
+++ b/console/build.sbt
@@ -13,6 +13,7 @@ dependsOn(
   Projects.semanticcpg,
   Projects.macros,
   Projects.jssrc2cpg,
+  Projects.pysrc2cpg,
   Projects.c2cpg % Test,
   Projects.x2cpg % "compile->compile;test->test"
 )
diff --git a/console/src/main/scala/io/joern/console/cpgcreation/PythonSrcCpgGenerator.scala b/console/src/main/scala/io/joern/console/cpgcreation/PythonSrcCpgGenerator.scala
index 3fb313e0979c..dd9069201f3e 100644
--- a/console/src/main/scala/io/joern/console/cpgcreation/PythonSrcCpgGenerator.scala
+++ b/console/src/main/scala/io/joern/console/cpgcreation/PythonSrcCpgGenerator.scala
@@ -1,9 +1,8 @@
 package io.joern.console.cpgcreation
 
 import io.joern.console.FrontendConfig
-import io.joern.x2cpg.passes.frontend.impl.{PythonTypeHintCallLinker, PythonTypeRecovery}
-import io.joern.x2cpg.passes.frontend.PythonNaiveCallLinker
 import io.shiftleft.codepropertygraph.Cpg
+import io.joern.pysrc2cpg.{PythonTypeRecovery, PythonTypeHintCallLinker, PythonNaiveCallLinker}
 
 import java.nio.file.Path
 
diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/frontend/PythonNaiveCallLinker.scala b/joern-cli/frontends/pysrc2cpg/src/main/scala/io/joern/pysrc2cpg/PythonNaiveCallLinker.scala
similarity index 96%
rename from joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/frontend/PythonNaiveCallLinker.scala
rename to joern-cli/frontends/pysrc2cpg/src/main/scala/io/joern/pysrc2cpg/PythonNaiveCallLinker.scala
index 0c5802edfb24..661a7dcb9cda 100644
--- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/frontend/PythonNaiveCallLinker.scala
+++ b/joern-cli/frontends/pysrc2cpg/src/main/scala/io/joern/pysrc2cpg/PythonNaiveCallLinker.scala
@@ -1,4 +1,4 @@
-package io.joern.x2cpg.passes.frontend
+package io.joern.pysrc2cpg
 
 import io.shiftleft.codepropertygraph.Cpg
 import io.shiftleft.codepropertygraph.generated.{EdgeTypes, PropertyNames}
diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/frontend/impl/PythonTypeHintCallLinker.scala b/joern-cli/frontends/pysrc2cpg/src/main/scala/io/joern/pysrc2cpg/PythonTypeHintCallLinker.scala
similarity index 93%
rename from joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/frontend/impl/PythonTypeHintCallLinker.scala
rename to joern-cli/frontends/pysrc2cpg/src/main/scala/io/joern/pysrc2cpg/PythonTypeHintCallLinker.scala
index bf70cd6b5ab7..f671e887d4b7 100644
--- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/frontend/impl/PythonTypeHintCallLinker.scala
+++ b/joern-cli/frontends/pysrc2cpg/src/main/scala/io/joern/pysrc2cpg/PythonTypeHintCallLinker.scala
@@ -1,4 +1,4 @@
-package io.joern.x2cpg.passes.frontend.impl
+package io.joern.pysrc2cpg
 
 import io.joern.x2cpg.passes.frontend.XTypeHintCallLinker
 import io.shiftleft.codepropertygraph.Cpg
diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/frontend/impl/PythonTypeRecovery.scala b/joern-cli/frontends/pysrc2cpg/src/main/scala/io/joern/pysrc2cpg/PythonTypeRecovery.scala
similarity index 87%
rename from joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/frontend/impl/PythonTypeRecovery.scala
rename to joern-cli/frontends/pysrc2cpg/src/main/scala/io/joern/pysrc2cpg/PythonTypeRecovery.scala
index f1acf2bbb5fa..693946b29a19 100644
--- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/frontend/impl/PythonTypeRecovery.scala
+++ b/joern-cli/frontends/pysrc2cpg/src/main/scala/io/joern/pysrc2cpg/PythonTypeRecovery.scala
@@ -1,4 +1,4 @@
-package io.joern.x2cpg.passes.frontend.impl
+package io.joern.pysrc2cpg
 
 import io.joern.x2cpg.Defines
 import io.joern.x2cpg.passes.frontend._
@@ -140,6 +140,15 @@ class SetPythonProcedureDefTask(node: CfgNode, symbolTable: SymbolTable[LocalKey
 class RecoverForPythonFile(cpg: Cpg, cu: File, builder: DiffGraphBuilder)
     extends RecoverForXCompilationUnit[File](cu, builder) {
 
+  /** Adds built-in functions to expect.
+    */
+  override def prepopulateSymbolTable(): Unit =
+    PythonTypeRecovery.BUILTINS
+      .map(t => (CallAlias(t), s"${PythonTypeRecovery.BUILTIN_PREFIX}.$t"))
+      .foreach { case (alias, typ) =>
+        symbolTable.put(alias, typ)
+      }
+
   override def importNodes(cu: AstNode): Traversal[CfgNode] = cu.ast.isCall.nameExact("import")
 
   override def postVisitImports(): Unit = {
@@ -243,3 +252,83 @@ class RecoverForPythonFile(cpg: Cpg, cu: File, builder: DiffGraphBuilder)
   }
 
 }
+
+object PythonTypeRecovery {
+
+  /** @see
+    *   <a href="https://docs.python.org/3/library/functions.html#func-dict">Python Built-in Functions</a>
+    */
+  lazy val BUILTINS: Set[String] = Set(
+    "abs",
+    "aiter",
+    "all",
+    "anext",
+    "ascii",
+    "bin",
+    "bool",
+    "breakpoint",
+    "bytearray",
+    "bytes",
+    "callable",
+    "chr",
+    "classmethod",
+    "compile",
+    "complex",
+    "delattr",
+    "dict",
+    "dir",
+    "divmod",
+    "enumerate",
+    "eval",
+    "exec",
+    "filter",
+    "float",
+    "format",
+    "frozenset",
+    "getattr",
+    "globals",
+    "hasattr",
+    "hash",
+    "help",
+    "hex",
+    "id",
+    "input",
+    "int",
+    "isinstance",
+    "issubclass",
+    "iter",
+    "len",
+    "list",
+    "locals",
+    "map",
+    "max",
+    "memoryview",
+    "min",
+    "next",
+    "object",
+    "oct",
+    "open",
+    "ord",
+    "pow",
+    "print",
+    "property",
+    "range",
+    "repr",
+    "reversed",
+    "round",
+    "set",
+    "setattr",
+    "slice",
+    "sorted",
+    "staticmethod",
+    "str",
+    "sum",
+    "super",
+    "tuple",
+    "type",
+    "vars",
+    "zip",
+    "__import__"
+  )
+  def BUILTIN_PREFIX = "builtins.py:<module>"
+}
diff --git a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/frontend/XTypeRecovery.scala b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/frontend/XTypeRecovery.scala
index bd9c02c71ff5..e2f70cee36dd 100644
--- a/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/frontend/XTypeRecovery.scala
+++ b/joern-cli/frontends/x2cpg/src/main/scala/io/joern/x2cpg/passes/frontend/XTypeRecovery.scala
@@ -7,7 +7,7 @@ import io.shiftleft.passes.CpgPass
 import io.shiftleft.semanticcpg.language._
 import io.shiftleft.semanticcpg.language.operatorextension.OpNodes
 import io.shiftleft.semanticcpg.language.operatorextension.OpNodes.Assignment
-import org.apache.logging.log4j.{LogManager, Logger}
+import org.slf4j.{Logger, LoggerFactory}
 import overflowdb.BatchedUpdate.DiffGraphBuilder
 import overflowdb.traversal.Traversal
 
@@ -97,7 +97,7 @@ abstract class ScopedXProcedure(val callingName: String, val fullName: String, v
   */
 abstract class SetXProcedureDefTask(node: CfgNode) extends RecursiveTask[Unit] {
 
-  protected val logger: Logger = LogManager.getLogger(classOf[SetXProcedureDefTask])
+  protected val logger: Logger = LoggerFactory.getLogger(classOf[SetXProcedureDefTask])
 
   override def compute(): Unit =
     node match {
@@ -140,10 +140,15 @@ abstract class RecoverForXCompilationUnit[ComputationalUnit <: AstNode](
     */
   protected val symbolTable = new SymbolTable[LocalKey](SBKey.fromNodeToLocalKey)
 
+  /** Provides an entrypoint to add known symbols and their possible types.
+    */
+  def prepopulateSymbolTable(): Unit = {}
+
   private def assignments: Traversal[Assignment] =
     cu.ast.isCall.name(Operators.assignment).map(new OpNodes.Assignment(_))
 
   override def compute(): Unit = try {
+    prepopulateSymbolTable()
     // Set known aliases that point to imports for local and external methods/modules
     setImportsFromDeclaredProcedures(importNodes(cu) ++ internalMethodNodes(cu))
     // Prune import names if the methods exist in the CPG
@@ -240,7 +245,7 @@ abstract class RecoverForXCompilationUnit[ComputationalUnit <: AstNode](
                 persistType(call, callTypes)(builder)
               }
             // Case 3: 'i' is the receiver for a field access on member 'f'
-            case (Some(call: Call), List(i: Identifier, f: FieldIdentifier))
+            case (Some(call: Call), List(i: Identifier, _: FieldIdentifier))
                 if call.name.equals(Operators.fieldAccess) =>
               persistType(i, symbolTable.get(x))(builder)
             // Case 4: We are elsewhere
@@ -277,8 +282,7 @@ abstract class SBKey {
 }
 
 object SBKey {
-  protected val logger: Logger = LogManager.getLogger(getClass)
-
+  protected val logger: Logger = LoggerFactory.getLogger(getClass)
   def fromNodeToLocalKey(node: AstNode): LocalKey = {
     node match {
       case n: FieldIdentifier => FieldVar(n.canonicalName)


Test patch:
diff --git a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/PySrc2CpgFixture.scala b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/PySrc2CpgFixture.scala
index 1218a1f421b2..006d338622ec 100644
--- a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/PySrc2CpgFixture.scala
+++ b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/PySrc2CpgFixture.scala
@@ -3,8 +3,6 @@ package io.joern.pysrc2cpg
 import io.joern.dataflowengineoss.layers.dataflows.{OssDataFlow, OssDataFlowOptions}
 import io.joern.dataflowengineoss.queryengine.EngineContext
 import io.joern.x2cpg.X2Cpg
-import io.joern.x2cpg.passes.frontend.impl.{PythonTypeHintCallLinker, PythonTypeRecovery}
-import io.joern.x2cpg.passes.frontend.PythonNaiveCallLinker
 import io.joern.x2cpg.testfixtures.{Code2CpgFixture, LanguageFrontend, TestCpg}
 import io.shiftleft.codepropertygraph.Cpg
 import io.shiftleft.semanticcpg.language.{ICallResolver, NoResolve}
diff --git a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/passes/TypeRecoveryPassTests.scala b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/passes/TypeRecoveryPassTests.scala
index 971d743c5789..f84dced6aeba 100644
--- a/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/passes/TypeRecoveryPassTests.scala
+++ b/joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pysrc2cpg/passes/TypeRecoveryPassTests.scala
@@ -135,6 +135,30 @@ class TypeRecoveryPassTests extends PySrc2CpgFixture(withOssDataflow = false) {
       val Some(columnConstructor) = cpg.call("Column").headOption
       columnConstructor.methodFullName shouldBe "flask_sqlalchemy.py:<module>.SQLAlchemy.Column.<init>"
     }
+
+    "recover paths for built-in calls" should {
+      lazy val cpg = code("""
+          |print("Hello world")
+          |max(1, 2)
+          |
+          |from foo import abs
+          |
+          |x = abs(-1)
+          |""".stripMargin)
+
+      "resolve 'print' and 'max' calls" in {
+        val Some(printCall) = cpg.call("print").headOption
+        printCall.methodFullName shouldBe "builtins.py:<module>.print"
+        val Some(maxCall) = cpg.call("max").headOption
+        maxCall.methodFullName shouldBe "builtins.py:<module>.max"
+      }
+
+      "select the imported abs over the built-in type when call is shadowed" in {
+        val Some(absCall) = cpg.call("abs").headOption
+        absCall.dynamicTypeHintFullName shouldBe Seq("foo.py:<module>.abs")
+      }
+
+    }
   }
 
 }