Merge branch 'main' into speed-up-type-resolution

cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonHandler.kt

@@ -49,4 +49,12 @@ abstract class PythonHandler<ResultNode : Node, HandlerNode : Python.AST.AST>(
     }
 
     abstract fun handleNode(node: HandlerNode): ResultNode
+
+    companion object {
+        /**
+         * A prefix to add to random names when handling for loops with multiple variables and
+         * having to add implicit assignments for the unwrapping process.
+         */
+        const val LOOP_VAR_PREFIX = "loopMultiVarHelperVar"
+    }
 }

cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt

@@ -235,7 +235,26 @@ class StatementHandler(frontend: PythonLanguageFrontend) :
         return ret
     }
 
-    private fun handleFor(node: Python.AST.NormalOrAsyncFor): Statement {
+    /**
+     * Translates a Python [`For`](https://docs.python.org/3/library/ast.html#ast.For) into an
+     * [ForEachStatement].
+     *
+     * Python supports implicit unpacking of multiple loop variables. To map this to CPG node, we
+     * translate the following implicit unpacking of code like this:
+     * ```python
+     * for a, b, c in someNestedList:
+     *     pass
+     * ```
+     *
+     * to have only one loop variable and add the unpacking statement to the top of the loop body
+     * like this:
+     * ```python
+     * for tempVar in someNestedList:
+     *     a, b, c = tempVar
+     *     pass
+     * ```
+     */
+    private fun handleFor(node: Python.AST.NormalOrAsyncFor): ForEachStatement {
         val ret = newForEachStatement(rawNode = node)
         if (node is IsAsync) {
             ret.addDeclaration(
@@ -247,18 +266,70 @@ class StatementHandler(frontend: PythonLanguageFrontend) :
         }
 
         ret.iterable = frontend.expressionHandler.handle(node.iter)
-        ret.variable = frontend.expressionHandler.handle(node.target)
-        ret.statement = makeBlock(node.body, parentNode = node)
+
+        when (val loopVar = frontend.expressionHandler.handle(node.target)) {
+            is InitializerListExpression -> { // unpacking
+                val (tempVarRef, unpackingAssignment) = getUnpackingNodes(loopVar)
+
+                ret.variable = tempVarRef
+
+                val body = makeBlock(node.body, parentNode = node)
+                body.statements.add(
+                    0,
+                    unpackingAssignment
+                ) // add the unpacking instruction to the top of the loop body
+                ret.statement = body
+            }
+            is Reference -> { // only one var
+                ret.variable = loopVar
+                ret.statement = makeBlock(node.body, parentNode = node)
+            }
+            else -> {
+                ret.variable =
+                    newProblemExpression(
+                        problem =
+                            "handleFor: cannot handle loop variable of type ${loopVar::class.simpleName}.",
+                        rawNode = node.target
+                    )
+                ret.statement = makeBlock(node.body, parentNode = node)
+            }
+        }
+
         if (node.orelse.isNotEmpty()) {
             ret.additionalProblems +=
                 newProblemExpression(
-                    problem = "Cannot handle \"orelse\" in for loops.",
+                    problem = "handleFor: Cannot handle \"orelse\" in for loops.",
                     rawNode = node
                 )
         }
         return ret
     }
 
+    /**
+     * This function creates two things:
+     * - A [Reference] to a variable with a random [Name]
+     * - An [AssignExpression] assigning the reference above to the [loopVar] input
+     *
+     * This is used in [handleFor] when loops have multiple loop variables to iterate over with
+     * automatic unpacking. We translate this implicit unpacking to multiple CPG nodes, as the CPG
+     * does not support automatic unpacking.
+     */
+    private fun getUnpackingNodes(
+        loopVar: InitializerListExpression
+    ): Pair<Reference, AssignExpression> {
+        val tempVarName = Name.random(prefix = LOOP_VAR_PREFIX)
+        val tempRef = newReference(name = tempVarName).implicit().codeAndLocationFrom(loopVar)
+        val assign =
+            newAssignExpression(
+                    operatorCode = "=",
+                    lhs = (loopVar).initializers,
+                    rhs = listOf(tempRef)
+                )
+                .implicit()
+                .codeAndLocationFrom(loopVar)
+        return Pair(tempRef, assign)
+    }
+
     private fun handleExpressionStatement(node: Python.AST.Expr): Statement {
         return frontend.expressionHandler.handle(node.value)
     }

cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/PythonAddDeclarationsPass.kt

@@ -159,14 +159,12 @@ class PythonAddDeclarationsPass(ctx: TranslationContext) : ComponentPass(ctx) {
         }
     }
 
-    // TODO document why this is necessary and implement for other possible places
+    // New variables can also be declared as `variable` in a [ForEachStatement]
     private fun handleForEach(node: ForEachStatement) {
-        when (node.variable) {
+        when (val forVar = node.variable) {
             is Reference -> {
-                val handled = handleReference(node.variable as Reference)
-                if (handled is Declaration) {
-                    handled.let { node.addDeclaration(it) }
-                }
+                val handled = handleReference(forVar)
+                (handled as? Declaration)?.let { scopeManager.addDeclaration(it) }
             }
         }
     }

cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt

@@ -805,8 +805,6 @@ class PythonFrontendTest : BaseTest() {
     }
 
     @Test
-    @Ignore // TODO fix & re-enable this test once there is proper support for multiple variables in
-    // a loop
     fun testIssue615() {
         val topLevel = Path.of("src", "test", "resources", "python")
         val tu =
@@ -818,58 +816,94 @@ class PythonFrontendTest : BaseTest() {
         val p = tu.namespaces["issue615"]
         assertNotNull(p)
 
-        assertEquals(1, p.declarations.size)
+        assertEquals(
+            5,
+            p.variables.size
+        ) // including one dummy variable introduced for the loop var
+        assertEquals(
+            4,
+            p.variables.filter { !it.name.localName.contains(PythonHandler.LOOP_VAR_PREFIX) }.size
+        )
         assertEquals(2, p.statements.size)
 
         // test = [(1, 2, 3)]
         val testDeclaration = p.variables[0]
         assertNotNull(testDeclaration)
         assertLocalName("test", testDeclaration)
-        val testDeclStmt = p.statements[0] as? DeclarationStatement
-        assertNotNull(testDeclStmt)
-        assertEquals(1, testDeclStmt.declarations.size)
-        assertEquals(testDeclaration, testDeclStmt.variables[0])
+        val testDeclStmt = p.statements[0]
+        assertIs<AssignExpression>(testDeclStmt)
 
         /* for loop:
         for t1, t2, t3 in test:
             print("bug ... {} {} {}".format(t1, t2, t3))
          */
-        val forStmt = p.statements[1] as? ForEachStatement
-        assertNotNull(forStmt)
-
-        val forVariable = forStmt.variable as? InitializerListExpression
-        assertNotNull(forVariable)
-        assertEquals(3, forVariable.initializers.size)
-        val t1Decl = forVariable.initializers[0] as? Reference
-        val t2Decl = forVariable.initializers[1] as? Reference
-        val t3Decl = forVariable.initializers[2] as? Reference
+        val forStmt = p.statements[1]
+        assertIs<ForEachStatement>(forStmt)
+
+        val forVariable = forStmt.variable
+        assertIs<Reference>(forVariable)
+        val forVarDecl =
+            p.declarations.firstOrNull {
+                it.name.localName.contains((PythonHandler.LOOP_VAR_PREFIX))
+            }
+        assertNotNull(forVarDecl)
+        assertRefersTo(forVariable, forVarDecl)
+
+        val iter = forStmt.iterable
+        assertIs<Reference>(iter)
+        assertRefersTo(iter, testDeclaration)
+
+        val forBody = forStmt.statement
+        assertIs<Block>(forBody)
+        assertEquals(2, forBody.statements.size) // loop var assign and print stmt
+
+        /*
+        We model the 3 loop variables
+
+        ```
+        for t1, t2, t3 in ...
+        ```
+
+        implicitly as follows:
+
+        ```
+        for tempVar in ...:
+          t1, t2, t3 = tempVar
+          rest of the loop
+        ```
+         */
+        val forVariableImplicitStmt = forBody.statements.firstOrNull()
+        assertIs<AssignExpression>(forVariableImplicitStmt)
+        assertEquals("=", forVariableImplicitStmt.operatorCode)
+        assertEquals(forStmt.variable, forVariableImplicitStmt.rhs.firstOrNull())
+        val (t1Decl, t2Decl, t3Decl) = forVariableImplicitStmt.declarations
+        val (t1RefAssign, t2RefAssign, t3RefAssign) = forVariableImplicitStmt.lhs
         assertNotNull(t1Decl)
         assertNotNull(t2Decl)
         assertNotNull(t3Decl)
-        // TODO no refersTo
-
-        val iter = forStmt.iterable as? Reference
-        assertNotNull(iter)
-        assertEquals(testDeclaration, iter.refersTo)
-
-        val forBody = forStmt.statement as? Block
-        assertNotNull(forBody)
-        assertEquals(1, forBody.statements.size)
+        assertIs<Reference>(t1RefAssign)
+        assertIs<Reference>(t2RefAssign)
+        assertIs<Reference>(t3RefAssign)
+        assertRefersTo(t1RefAssign, t1Decl)
+        assertRefersTo(t2RefAssign, t2Decl)
+        assertRefersTo(t3RefAssign, t3Decl)
 
         // print("bug ... {} {} {}".format(t1, t2, t3))
-        val forBodyStmt = forBody.statements[0] as? CallExpression
+        val forBodyStmt = forBody.statements<CallExpression>(1)
         assertNotNull(forBodyStmt)
         assertLocalName("print", forBodyStmt)
 
-        val printArg = forBodyStmt.arguments[0] as? MemberCallExpression
-        assertNotNull(printArg)
-        val formatArgT1 = printArg.arguments[0] as? Reference
-        assertNotNull(formatArgT1)
-        val formatArgT2 = printArg.arguments[1] as? Reference
-        assertNotNull(formatArgT2)
-        val formatArgT3 = printArg.arguments[2] as? Reference
-        assertNotNull(formatArgT3)
-        // TODO check refersTo
+        val printArg = forBodyStmt.arguments[0]
+        assertIs<MemberCallExpression>(printArg)
+        val formatArgT1 = printArg.arguments[0]
+        assertIs<Reference>(formatArgT1)
+        assertRefersTo(formatArgT1, t1Decl)
+        val formatArgT2 = printArg.arguments[1]
+        assertIs<Reference>(formatArgT2)
+        assertRefersTo(formatArgT2, t2Decl)
+        val formatArgT3 = printArg.arguments[2]
+        assertIs<Reference>(formatArgT3)
+        assertRefersTo(formatArgT3, t3Decl)
     }
 
     @Test