Merge remote-tracking branch 'origin/main' into ak/pythonOrElse

buildSrc/src/main/kotlin/cpg.common-conventions.gradle.kts

@@ -107,7 +107,7 @@ kotlin {
 
 tasks.withType<KotlinCompile> {
     compilerOptions {
-        freeCompilerArgs = listOf("-opt-in=kotlin.RequiresOptIn", "-Xcontext-receivers")
+        freeCompilerArgs = listOf("-opt-in=kotlin.RequiresOptIn", "-opt-in=kotlin.uuid.ExperimentalUuidApi", "-Xcontext-receivers")
     }
 }
 

cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Name.kt

@@ -29,6 +29,7 @@ import de.fraunhofer.aisec.cpg.frontends.Handler
 import de.fraunhofer.aisec.cpg.frontends.Language
 import de.fraunhofer.aisec.cpg.frontends.LanguageFrontend
 import java.util.*
+import kotlin.uuid.Uuid
 
 /**
  * This class represents anything that can have a "Name". In the simplest case it only represents a
@@ -49,6 +50,18 @@ class Name(
         language: Language<*>?
     ) : this(localName, parent, language?.namespaceDelimiter ?: ".")
 
+    companion object {
+        /**
+         * Creates a random name starting with a prefix plus a random UUID (version 4). The Name is
+         * prefixed by [prefix], followed by a separator character [separatorChar] and finalized by
+         * a random UUID ("-" separators also replaced with [separatorChar]).
+         */
+        fun random(prefix: String, separatorChar: Char = '_'): Name {
+            val randomPart = Uuid.random().toString().replace('-', separatorChar)
+            return Name(localName = prefix + separatorChar + randomPart)
+        }
+    }
+
     /**
      * The full string representation of this name. Since [localName] and [parent] are immutable,
      * this is basically a cache for [toString]. Otherwise, we would need to call [toString] a lot

cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/collections/EdgeList.kt

@@ -100,6 +100,16 @@ abstract class EdgeList<NodeType : Node, EdgeType : Edge<NodeType>>(
         edges.forEach { handleOnRemove(it) }
     }
 
+    /**
+     * This function creates a new edge (of [EdgeType]) to/from the specified node [target]
+     * (depending on [outgoing]) and adds it to the specified index in the list.
+     */
+    fun add(index: Int, target: NodeType) {
+        val edge = createEdge(target, init, this.outgoing)
+
+        return add(index, edge)
+    }
+
     override fun add(index: Int, element: EdgeType) {
         // Make sure, the index is always set
         element.index = this.size

cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/collections/UnwrappedEdgeList.kt

@@ -40,7 +40,7 @@ class UnwrappedEdgeList<NodeType : Node, EdgeType : Edge<NodeType>>(
 ) : UnwrappedEdgeCollection<NodeType, EdgeType>(list), MutableList<NodeType> {
 
     override fun add(index: Int, element: NodeType) {
-        TODO("Not yet implemented")
+        return list.add(index, element)
     }
 
     override fun addAll(index: Int, elements: Collection<NodeType>): Boolean {

cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/DFGPass.kt

@@ -459,7 +459,6 @@ class DFGPass(ctx: TranslationContext) : ComponentPass(ctx) {
     fun handleCallExpression(call: CallExpression, inferDfgForUnresolvedSymbols: Boolean) {
         // Remove existing DFG edges since they are no longer valid (e.g. after updating the
         // CallExpression with the invokes edges to the called functions)
-        call.prevDFG.forEach { it.nextDFGEdges.removeIf { edge -> edge.end == call } }
         call.prevDFGEdges.clear()
 
         if (call.invokes.isEmpty() && inferDfgForUnresolvedSymbols) {

cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/edges/collections/EdgeListTest.kt

@@ -51,7 +51,8 @@ class EdgeListTest {
                 assertEquals(i, edge.index, "index mismatch $i != ${edge.index}")
             }
 
-            // insert something at position 1, this should shift the existing two entries + 1
+            // insert something at position 1, this should shift the existing entries (after the
+            // position) + 1
             list.add(1, AstEdge(node1, node4))
             assertEquals(3, list.size)
 

cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/edges/collections/UnwrappedEdgeListTest.kt

@@ -26,6 +26,9 @@
 package de.fraunhofer.aisec.cpg.graph.edges.collections
 
 import de.fraunhofer.aisec.cpg.frontends.TestLanguageFrontend
+import de.fraunhofer.aisec.cpg.graph.Node
+import de.fraunhofer.aisec.cpg.graph.edges.ast.AstEdge
+import de.fraunhofer.aisec.cpg.graph.edges.ast.AstEdges
 import de.fraunhofer.aisec.cpg.graph.newLiteral
 import kotlin.test.Test
 import kotlin.test.assertEquals
@@ -53,6 +56,39 @@ class UnwrappedEdgeListTest {
         }
     }
 
+    @Test
+    fun testAddIndex() {
+        with(TestLanguageFrontend()) {
+            var node1 = newLiteral(1)
+            var node2 = newLiteral(2)
+            var node3 = newLiteral(3)
+            var node4 = newLiteral(4)
+
+            var list = AstEdges<Node, AstEdge<Node>>(thisRef = node1)
+            list += node2
+            list += node3
+
+            assertEquals(2, list.size)
+            list.forEachIndexed { i, edge ->
+                assertEquals(i, edge.index, "index mismatch $i != ${edge.index}")
+            }
+
+            // insert something at position 1 (using the unwrapped list), this should shift the
+            // existing entries (after the position) + 1
+            var unwrapped = list.unwrap()
+            unwrapped.add(1, node4)
+            assertEquals(3, list.size)
+
+            // indices should still be in sync afterward
+            list.forEachIndexed { i, edge ->
+                assertEquals(i, edge.index, "index mismatch $i != ${edge.index}")
+            }
+
+            // the order should be node2, node4, node3
+            assertEquals<List<Node>>(listOf(node2, node4, node3), unwrapped)
+        }
+    }
+
     @Test
     fun testIterator() {
         with(TestLanguageFrontend()) {

cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandler.kt

@@ -84,9 +84,8 @@ class ExpressionHandler(frontend: PythonLanguageFrontend) :
     }
 
     /**
-     * Translates a Python (Named
-     * Expression)[https://docs.python.org/3/library/ast.html#ast.NamedExpr] into an
-     * [AssignExpression].
+     * Translates a Python [`NamedExpr`](https://docs.python.org/3/library/ast.html#ast.NamedExpr)
+     * into an [AssignExpression].
      *
      * As opposed to the Assign node, both target and value must be single nodes.
      */
@@ -181,22 +180,42 @@ class ExpressionHandler(frontend: PythonLanguageFrontend) :
         return subscriptExpression
     }
 
+    /**
+     * This method handles the python
+     * [`BoolOp`](https://docs.python.org/3/library/ast.html#ast.BoolOp).
+     *
+     * Generates a (potentially nested) [BinaryOperator] from a `BoolOp`. Less than two operands in
+     * [Python.AST.BoolOp.values] don't make sense and will generate a [ProblemExpression]. If only
+     * two operands exist, a simple [BinaryOperator] will be generated. More than two operands will
+     * lead to a nested [BinaryOperator]. E.g., if [Python.AST.BoolOp.values] contains the operators
+     * `[a, b, c]`, the result will be `a OP (b OP c)`.
+     */
     private fun handleBoolOp(node: Python.AST.BoolOp): Expression {
         val op =
             when (node.op) {
                 is Python.AST.And -> "and"
                 is Python.AST.Or -> "or"
             }
-        val ret = newBinaryOperator(operatorCode = op, rawNode = node)
-        if (node.values.size != 2) {
-            return newProblemExpression(
-                "Expected exactly two expressions but got " + node.values.size,
+
+        return if (node.values.size <= 1) {
+            newProblemExpression(
+                "Expected exactly two expressions but got ${node.values.size}",
                 rawNode = node
             )
+        } else {
+            // Start with the last two operands, then keep prepending the previous ones until the
+            // list is finished.
+            val lastTwo = newBinaryOperator(op, rawNode = node)
+            lastTwo.rhs = handle(node.values.last())
+            lastTwo.lhs = handle(node.values[node.values.size - 2])
+            return node.values.subList(0, node.values.size - 2).foldRight(lastTwo) { newVal, start
+                ->
+                val nextValue = newBinaryOperator(op, rawNode = node)
+                nextValue.rhs = start
+                nextValue.lhs = handle(newVal)
+                nextValue
+            }
         }
-        ret.lhs = handle(node.values[0])
-        ret.rhs = handle(node.values[1])
-        return ret
     }
 
     private fun handleList(node: Python.AST.List): Expression {

cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonLanguageFrontend.kt

@@ -144,24 +144,7 @@ class PythonLanguageFrontend(language: Language<PythonLanguageFrontend>, ctx: Tr
                 autoType()
             }
             is Python.AST.Name -> {
-                // We have some kind of name here; let's quickly check, if this is a primitive type
-                val id = type.id
-                if (id in language.primitiveTypeNames) {
-                    return primitiveType(id)
-                }
-
-                // Otherwise, this could already be a fully qualified type
-                val name =
-                    if (language.namespaceDelimiter in id) {
-                        // TODO: This might create problem with nested classes
-                        parseName(id)
-                    } else {
-                        // otherwise, we can just simply take the unqualified name and the type
-                        // resolver will take care of the rest
-                        id
-                    }
-
-                objectType(name)
+                this.typeOf(type.id)
             }
             else -> {
                 // The AST supplied us with some kind of type information, but we could not parse
@@ -171,6 +154,21 @@ class PythonLanguageFrontend(language: Language<PythonLanguageFrontend>, ctx: Tr
         }
     }
 
+    /** Resolves a [Type] based on its string identifier. */
+    fun typeOf(typeId: String): Type {
+        // Check if the typeId contains a namespace delimiter for qualified types
+        val name =
+            if (language.namespaceDelimiter in typeId) {
+                // TODO: This might create problem with nested classes
+                parseName(typeId)
+            } else {
+                // Unqualified name, resolved by the type resolver
+                typeId
+            }
+
+        return objectType(name)
+    }
+
     /**
      * This functions extracts the source code from the input file given a location. This is a bit
      * tricky in Python, as indents are part of the syntax. We also don't want to include leading

cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt

@@ -35,10 +35,7 @@ import de.fraunhofer.aisec.cpg.graph.*
 import de.fraunhofer.aisec.cpg.graph.Annotation
 import de.fraunhofer.aisec.cpg.graph.declarations.*
 import de.fraunhofer.aisec.cpg.graph.statements.*
-import de.fraunhofer.aisec.cpg.graph.statements.expressions.Block
-import de.fraunhofer.aisec.cpg.graph.statements.expressions.Expression
-import de.fraunhofer.aisec.cpg.graph.statements.expressions.MemberExpression
-import de.fraunhofer.aisec.cpg.graph.statements.expressions.ProblemExpression
+import de.fraunhofer.aisec.cpg.graph.statements.expressions.*
 import de.fraunhofer.aisec.cpg.graph.types.FunctionType
 import de.fraunhofer.aisec.cpg.helpers.Util
 import kotlin.collections.plusAssign
@@ -126,6 +123,7 @@ class StatementHandler(frontend: PythonLanguageFrontend) :
 
         return tryStatement
     }
+
     /**
      * Translates a Python [`Assert`](https://docs.python.org/3/library/ast.html#ast.Assert) into a
      * [AssertStatement].
@@ -239,18 +237,15 @@ class StatementHandler(frontend: PythonLanguageFrontend) :
         return frontend.expressionHandler.handle(node.value)
     }
 
-    private fun handleAnnAssign(node: Python.AST.AnnAssign): Statement {
-        // TODO: annotations
+    /**
+     * Translates a Python [`AnnAssign`](https://docs.python.org/3/library/ast.html#ast.AnnAssign)
+     * into an [AssignExpression].
+     */
+    private fun handleAnnAssign(node: Python.AST.AnnAssign): AssignExpression {
         val lhs = frontend.expressionHandler.handle(node.target)
-        return if (node.value != null) {
-            newAssignExpression(
-                lhs = listOf(lhs),
-                rhs = listOf(frontend.expressionHandler.handle(node.value!!)), // TODO !!
-                rawNode = node
-            )
-        } else {
-            lhs
-        }
+        lhs.type = frontend.typeOf(node.annotation)
+        val rhs = node.value?.let { listOf(frontend.expressionHandler.handle(it)) } ?: emptyList()
+        return newAssignExpression(lhs = listOf(lhs), rhs = rhs, rawNode = node)
     }
 
     private fun handleIf(node: Python.AST.If): Statement {
@@ -277,8 +272,16 @@ class StatementHandler(frontend: PythonLanguageFrontend) :
         return ret
     }
 
-    private fun handleAssign(node: Python.AST.Assign): Statement {
+    /**
+     * Translates a Python [`Assign`](https://docs.python.org/3/library/ast.html#ast.Assign) into an
+     * [AssignExpression].
+     */
+    private fun handleAssign(node: Python.AST.Assign): AssignExpression {
         val lhs = node.targets.map { frontend.expressionHandler.handle(it) }
+        node.type_comment?.let { typeComment ->
+            val tpe = frontend.typeOf(typeComment)
+            lhs.forEach { it.type = tpe }
+        }
         val rhs = frontend.expressionHandler.handle(node.value)
         if (rhs is List<*>)
             newAssignExpression(