From 79fa1a9c4cb34589bc7bf56cb3b0ab73c2bb49ca Mon Sep 17 00:00:00 2001 From: Christian Banse Date: Wed, 25 Sep 2024 12:15:25 +0200 Subject: [PATCH 01/30] Added `add(index, element)` for `UnwrappedEdgeList` This adds support for adding nodes at a specific location in an unwrapped edge list. --- .../aisec/cpg/graph/edges/collections/EdgeList.kt | 10 ++++++++++ .../cpg/graph/edges/collections/UnwrappedEdgeList.kt | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/collections/EdgeList.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/collections/EdgeList.kt index cfbf9a8c03..f4384112ca 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/collections/EdgeList.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/collections/EdgeList.kt @@ -100,6 +100,16 @@ abstract class EdgeList>( edges.forEach { handleOnRemove(it) } } + /** + * This function creates a new edge (of [EdgeType]) to/from the specified node [target] + * (depending on [outgoing]) and adds it to the specified index in the list. + */ + fun add(index: Int, target: NodeType) { + val edge = createEdge(target, init, this.outgoing) + + return add(index, edge) + } + override fun add(index: Int, element: EdgeType) { // Make sure, the index is always set element.index = this.size diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/collections/UnwrappedEdgeList.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/collections/UnwrappedEdgeList.kt index d51499a84f..e46f14c9b7 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/collections/UnwrappedEdgeList.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/edges/collections/UnwrappedEdgeList.kt @@ -40,7 +40,7 @@ class UnwrappedEdgeList>( ) : UnwrappedEdgeCollection(list), MutableList { override fun add(index: Int, element: NodeType) { - TODO("Not yet implemented") + return list.add(index, element) } override fun addAll(index: Int, elements: Collection): Boolean { From dc293e644d64847df241306b0b036c1eb05030e7 Mon Sep 17 00:00:00 2001 From: Christian Banse Date: Wed, 25 Sep 2024 12:25:54 +0200 Subject: [PATCH 02/30] Added test case --- .../graph/edges/collections/EdgeListTest.kt | 3 +- .../collections/UnwrappedEdgeListTest.kt | 36 +++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/edges/collections/EdgeListTest.kt b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/edges/collections/EdgeListTest.kt index 6fce43bc16..500dc8ea53 100644 --- a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/edges/collections/EdgeListTest.kt +++ b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/edges/collections/EdgeListTest.kt @@ -51,7 +51,8 @@ class EdgeListTest { assertEquals(i, edge.index, "index mismatch $i != ${edge.index}") } - // insert something at position 1, this should shift the existing two entries + 1 + // insert something at position 1, this should shift the existing entries (after the + // position) + 1 list.add(1, AstEdge(node1, node4)) assertEquals(3, list.size) diff --git a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/edges/collections/UnwrappedEdgeListTest.kt b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/edges/collections/UnwrappedEdgeListTest.kt index 30bdc67838..2d318ae5a6 100644 --- a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/edges/collections/UnwrappedEdgeListTest.kt +++ b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/edges/collections/UnwrappedEdgeListTest.kt @@ -26,6 +26,9 @@ package de.fraunhofer.aisec.cpg.graph.edges.collections import de.fraunhofer.aisec.cpg.frontends.TestLanguageFrontend +import de.fraunhofer.aisec.cpg.graph.Node +import de.fraunhofer.aisec.cpg.graph.edges.ast.AstEdge +import de.fraunhofer.aisec.cpg.graph.edges.ast.AstEdges import de.fraunhofer.aisec.cpg.graph.newLiteral import kotlin.test.Test import kotlin.test.assertEquals @@ -53,6 +56,39 @@ class UnwrappedEdgeListTest { } } + @Test + fun testAddIndex() { + with(TestLanguageFrontend()) { + var node1 = newLiteral(1) + var node2 = newLiteral(2) + var node3 = newLiteral(3) + var node4 = newLiteral(4) + + var list = AstEdges>(thisRef = node1) + list += node2 + list += node3 + + assertEquals(2, list.size) + list.forEachIndexed { i, edge -> + assertEquals(i, edge.index, "index mismatch $i != ${edge.index}") + } + + // insert something at position 1 (using the unwrapped list), this should shift the + // existing entries (after the position) + 1 + var unwrapped = list.unwrap() + unwrapped.add(1, node4) + assertEquals(3, list.size) + + // indices should still be in sync afterward + list.forEachIndexed { i, edge -> + assertEquals(i, edge.index, "index mismatch $i != ${edge.index}") + } + + // the order should be node2, node4, node3 + assertEquals>(listOf(node2, node4, node3), unwrapped) + } + } + @Test fun testIterator() { with(TestLanguageFrontend()) { From 715d36037d3dce1264da310a51b94c3d2c534203 Mon Sep 17 00:00:00 2001 From: Maximilian Kaul Date: Wed, 25 Sep 2024 12:45:58 +0200 Subject: [PATCH 03/30] Bump Kotlin to 2.0.20 --- gradle/libs.versions.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 9ae3851b12..c4ab5fff09 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -1,5 +1,5 @@ [versions] -kotlin = "2.0.0" +kotlin = "2.0.20" kotlin19 = "1.9.10" neo4j = "4.0.10" log4j = "2.24.0" From 441dca3d414e21013a28dbf4194ae631f36ac49e Mon Sep 17 00:00:00 2001 From: Maximilian Kaul Date: Wed, 25 Sep 2024 13:21:35 +0200 Subject: [PATCH 04/30] Handler: add get a random Name --- .../kotlin/de/fraunhofer/aisec/cpg/frontends/Handler.kt | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/Handler.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/Handler.kt index 7d5b45a2a6..68d0ae5da3 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/Handler.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/Handler.kt @@ -30,6 +30,7 @@ import de.fraunhofer.aisec.cpg.helpers.Util.errorWithFileLocation import java.lang.reflect.ParameterizedType import java.lang.reflect.Type import java.util.function.Supplier +import kotlin.uuid.Uuid import org.slf4j.Logger import org.slf4j.LoggerFactory @@ -166,6 +167,14 @@ abstract class Handler Date: Wed, 25 Sep 2024 13:23:33 +0200 Subject: [PATCH 05/30] implement multi vars in for loops --- .../cpg/frontends/python/StatementHandler.kt | 47 +++++++++++++- .../cpg/passes/PythonAddDeclarationsPass.kt | 8 +-- .../frontends/python/PythonFrontendTest.kt | 64 +++++++++++++------ 3 files changed, 93 insertions(+), 26 deletions(-) diff --git a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt index 10e6bf846d..0093b93319 100644 --- a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt +++ b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt @@ -36,10 +36,13 @@ import de.fraunhofer.aisec.cpg.graph.declarations.* import de.fraunhofer.aisec.cpg.graph.statements.AssertStatement import de.fraunhofer.aisec.cpg.graph.statements.DeclarationStatement import de.fraunhofer.aisec.cpg.graph.statements.Statement +import de.fraunhofer.aisec.cpg.graph.statements.expressions.AssignExpression import de.fraunhofer.aisec.cpg.graph.statements.expressions.Block import de.fraunhofer.aisec.cpg.graph.statements.expressions.Expression +import de.fraunhofer.aisec.cpg.graph.statements.expressions.InitializerListExpression import de.fraunhofer.aisec.cpg.graph.statements.expressions.MemberExpression import de.fraunhofer.aisec.cpg.graph.statements.expressions.ProblemExpression +import de.fraunhofer.aisec.cpg.graph.statements.expressions.Reference import de.fraunhofer.aisec.cpg.graph.types.FunctionType import de.fraunhofer.aisec.cpg.helpers.Util import kotlin.collections.plusAssign @@ -180,18 +183,56 @@ class StatementHandler(frontend: PythonLanguageFrontend) : } ret.iterable = frontend.expressionHandler.handle(node.iter) - ret.variable = frontend.expressionHandler.handle(node.target) - ret.statement = makeBlock(node.body, parentNode = node) + val loopVar = frontend.expressionHandler.handle(node.target) + + when (loopVar) { + is InitializerListExpression -> { + + val (dummyVarRef, dummyAssign) = getDummyAssign(loopVar) + + ret.variable = dummyVarRef + val body = makeBlock(node.body, parentNode = node) + + body.statements.add(0, dummyAssign) + + ret.statement = body + } + is Reference -> { + ret.variable = loopVar + ret.statement = makeBlock(node.body, parentNode = node) + } + else -> { + TODO() + } + } + if (node.orelse.isNotEmpty()) { ret.additionalProblems += newProblemExpression( - problem = "Cannot handle \"orelse\" in for loops.", + problem = "handleFor: Cannot handle \"orelse\" in for loops.", rawNode = node ) } return ret } + /** TODO */ + private fun getDummyAssign( + loopVar: InitializerListExpression + ): Pair { + val tempVarName = getRandomTempName() + val tempRef = newReference(name = tempVarName) + tempRef.isImplicit = true + val assign = + newAssignExpression( + operatorCode = "=", + lhs = (loopVar).initializers, + rhs = listOf(tempRef) + ) + assign.isImplicit = true + return Pair(tempRef, assign) + } + private fun handleExpressionStatement(node: Python.AST.Expr): Statement { return frontend.expressionHandler.handle(node.value) } diff --git a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/PythonAddDeclarationsPass.kt b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/PythonAddDeclarationsPass.kt index 28a6dec744..aae0af5991 100644 --- a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/PythonAddDeclarationsPass.kt +++ b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/PythonAddDeclarationsPass.kt @@ -161,12 +161,10 @@ class PythonAddDeclarationsPass(ctx: TranslationContext) : ComponentPass(ctx) { // TODO document why this is necessary and implement for other possible places private fun handleForEach(node: ForEachStatement) { - when (node.variable) { + when (val forVar = node.variable) { is Reference -> { - val handled = handleReference(node.variable as Reference) - if (handled is Declaration) { - handled.let { node.addDeclaration(it) } - } + val handled = handleReference(forVar) + (handled as? Declaration)?.let { forVar.addDeclaration(it) } } } } diff --git a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt index 5d63e06d6d..b433b1dd80 100644 --- a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt +++ b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt @@ -805,8 +805,6 @@ class PythonFrontendTest : BaseTest() { } @Test - @Ignore // TODO fix & re-enable this test once there is proper support for multiple variables in - // a loop fun testIssue615() { val topLevel = Path.of("src", "test", "resources", "python") val tu = @@ -818,17 +816,19 @@ class PythonFrontendTest : BaseTest() { val p = tu.namespaces["issue615"] assertNotNull(p) - assertEquals(1, p.declarations.size) + assertEquals( + 5, + p.variables.size + ) // including one dummy variable introduced for the loop var + assertEquals(4, p.variables.filter { !it.name.localName.contains("TEMP-RANDOM-NAME") }.size) assertEquals(2, p.statements.size) // test = [(1, 2, 3)] val testDeclaration = p.variables[0] assertNotNull(testDeclaration) assertLocalName("test", testDeclaration) - val testDeclStmt = p.statements[0] as? DeclarationStatement + val testDeclStmt = p.statements[0] as? AssignExpression assertNotNull(testDeclStmt) - assertEquals(1, testDeclStmt.declarations.size) - assertEquals(testDeclaration, testDeclStmt.variables[0]) /* for loop: for t1, t2, t3 in test: @@ -837,16 +837,11 @@ class PythonFrontendTest : BaseTest() { val forStmt = p.statements[1] as? ForEachStatement assertNotNull(forStmt) - val forVariable = forStmt.variable as? InitializerListExpression + val forVariable = forStmt.variable as? Reference assertNotNull(forVariable) - assertEquals(3, forVariable.initializers.size) - val t1Decl = forVariable.initializers[0] as? Reference - val t2Decl = forVariable.initializers[1] as? Reference - val t3Decl = forVariable.initializers[2] as? Reference - assertNotNull(t1Decl) - assertNotNull(t2Decl) - assertNotNull(t3Decl) - // TODO no refersTo + val forVarDecl = forVariable.declarations.first() + assertEquals(1, forVariable.declarations.size) + assertEquals(forVarDecl, forVariable.refersTo) val iter = forStmt.iterable as? Reference assertNotNull(iter) @@ -854,10 +849,41 @@ class PythonFrontendTest : BaseTest() { val forBody = forStmt.statement as? Block assertNotNull(forBody) - assertEquals(1, forBody.statements.size) + assertEquals(2, forBody.statements.size) // loop var assign and print stmt + + /* + We model the 3 loop variables + + ``` + for t1, t2, t3 in ... + ``` + + implicitly as follows: + + ``` + for tempVar in ...: + t1, t2, t3 = tempVar + rest of the loop + ``` + */ + val forVariableImplicitStmt = forBody.statements.first() as? AssignExpression + assertNotNull(forVariableImplicitStmt) + assertEquals("=", forVariableImplicitStmt.operatorCode) + assertEquals(forStmt.variable, forVariableImplicitStmt.rhs.first()) + val (t1Decl, t2Decl, t3Decl) = forVariableImplicitStmt.declarations + val (t1RefAssign, t2RefAssign, t3RefAssign) = forVariableImplicitStmt.lhs + assertNotNull(t1Decl) + assertNotNull(t2Decl) + assertNotNull(t3Decl) + assertNotNull(t1RefAssign as? Reference) + assertNotNull(t2RefAssign as? Reference) + assertNotNull(t3RefAssign as? Reference) + assertEquals(t1Decl, t1RefAssign.refersTo) + assertEquals(t2Decl, t2RefAssign.refersTo) + assertEquals(t3Decl, t3RefAssign.refersTo) // print("bug ... {} {} {}".format(t1, t2, t3)) - val forBodyStmt = forBody.statements[0] as? CallExpression + val forBodyStmt = forBody.statements[1] as? CallExpression assertNotNull(forBodyStmt) assertLocalName("print", forBodyStmt) @@ -865,11 +891,13 @@ class PythonFrontendTest : BaseTest() { assertNotNull(printArg) val formatArgT1 = printArg.arguments[0] as? Reference assertNotNull(formatArgT1) + assertEquals(t1Decl, formatArgT1.refersTo) val formatArgT2 = printArg.arguments[1] as? Reference assertNotNull(formatArgT2) + assertEquals(t2Decl, formatArgT2.refersTo) val formatArgT3 = printArg.arguments[2] as? Reference assertNotNull(formatArgT3) - // TODO check refersTo + assertEquals(t3Decl, formatArgT3.refersTo) } @Test From 6ef5616e6f4c2be6715f4de89ef22b04b38cf8ff Mon Sep 17 00:00:00 2001 From: Maximilian Kaul Date: Wed, 25 Sep 2024 13:26:22 +0200 Subject: [PATCH 06/30] enable uuid for the whole project --- buildSrc/src/main/kotlin/cpg.common-conventions.gradle.kts | 2 +- .../main/kotlin/de/fraunhofer/aisec/cpg/frontends/Handler.kt | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/buildSrc/src/main/kotlin/cpg.common-conventions.gradle.kts b/buildSrc/src/main/kotlin/cpg.common-conventions.gradle.kts index 7f6c09c47c..b7e2fad253 100644 --- a/buildSrc/src/main/kotlin/cpg.common-conventions.gradle.kts +++ b/buildSrc/src/main/kotlin/cpg.common-conventions.gradle.kts @@ -107,7 +107,7 @@ kotlin { tasks.withType { compilerOptions { - freeCompilerArgs = listOf("-opt-in=kotlin.RequiresOptIn", "-Xcontext-receivers") + freeCompilerArgs = listOf("-opt-in=kotlin.RequiresOptIn", "-opt-in=kotlin.uuid.ExperimentalUuidApi", "-Xcontext-receivers") } } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/Handler.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/Handler.kt index 68d0ae5da3..dd5cf76043 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/Handler.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/Handler.kt @@ -168,7 +168,6 @@ abstract class Handler Date: Wed, 25 Sep 2024 13:51:39 +0200 Subject: [PATCH 07/30] move helper function to Name --- .../kotlin/de/fraunhofer/aisec/cpg/frontends/Handler.kt | 8 -------- .../main/kotlin/de/fraunhofer/aisec/cpg/graph/Name.kt | 9 +++++++++ 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/Handler.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/Handler.kt index dd5cf76043..7d5b45a2a6 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/Handler.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/Handler.kt @@ -30,7 +30,6 @@ import de.fraunhofer.aisec.cpg.helpers.Util.errorWithFileLocation import java.lang.reflect.ParameterizedType import java.lang.reflect.Type import java.util.function.Supplier -import kotlin.uuid.Uuid import org.slf4j.Logger import org.slf4j.LoggerFactory @@ -167,13 +166,6 @@ abstract class Handler? ) : this(localName, parent, language?.namespaceDelimiter ?: ".") + companion object { + /** Creates a random name starting with a prefix plus a random UUID (version 4). */ + fun getRandomTempName(prefix: String = "TEMP-RANDOM-NAME"): Name { + val randomPart = Uuid.random().toString().replace("-", "_") + return Name(localName = prefix + "_" + randomPart) + } + } + /** * The full string representation of this name. Since [localName] and [parent] are immutable, * this is basically a cache for [toString]. Otherwise, we would need to call [toString] a lot From ef922eb47d95b348b59f3173859f8d4e2ebb3afa Mon Sep 17 00:00:00 2001 From: Maximilian Kaul Date: Wed, 25 Sep 2024 14:04:47 +0200 Subject: [PATCH 08/30] configure prefix and separator char --- .../kotlin/de/fraunhofer/aisec/cpg/graph/Name.kt | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Name.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Name.kt index ca0ecb75e6..0606d03cbb 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Name.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Name.kt @@ -51,10 +51,14 @@ class Name( ) : this(localName, parent, language?.namespaceDelimiter ?: ".") companion object { - /** Creates a random name starting with a prefix plus a random UUID (version 4). */ - fun getRandomTempName(prefix: String = "TEMP-RANDOM-NAME"): Name { - val randomPart = Uuid.random().toString().replace("-", "_") - return Name(localName = prefix + "_" + randomPart) + /** + * Creates a random name starting with a prefix plus a random UUID (version 4). The Name is + * prefixed by [prefix], followed by a separator character [separatorChar] and finalized by + * a random UUID ("-" separators also replaced with [separatorChar]). + */ + fun getRandomTempName(prefix: String, separatorChar: Char = '_'): Name { + val randomPart = Uuid.random().toString().replace('-', separatorChar) + return Name(localName = prefix + separatorChar + randomPart) } } From 30a7d1f363ae063f7cbccf68ad7f33d469be3021 Mon Sep 17 00:00:00 2001 From: Maximilian Kaul Date: Wed, 25 Sep 2024 14:06:34 +0200 Subject: [PATCH 09/30] use the helper function now located in Name --- .../aisec/cpg/frontends/python/StatementHandler.kt | 2 +- .../aisec/cpg/frontends/python/PythonFrontendTest.kt | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt index 7477b690b5..324fa99a7a 100644 --- a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt +++ b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt @@ -220,7 +220,7 @@ class StatementHandler(frontend: PythonLanguageFrontend) : private fun getDummyAssign( loopVar: InitializerListExpression ): Pair { - val tempVarName = getRandomTempName() + val tempVarName = Name.getRandomTempName(prefix = "loopMultiVarHelperVar") val tempRef = newReference(name = tempVarName) tempRef.isImplicit = true val assign = diff --git a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt index b433b1dd80..f211598669 100644 --- a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt +++ b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt @@ -820,7 +820,10 @@ class PythonFrontendTest : BaseTest() { 5, p.variables.size ) // including one dummy variable introduced for the loop var - assertEquals(4, p.variables.filter { !it.name.localName.contains("TEMP-RANDOM-NAME") }.size) + assertEquals( + 4, + p.variables.filter { !it.name.localName.contains("loopMultiVarHelperVar") }.size + ) assertEquals(2, p.statements.size) // test = [(1, 2, 3)] From 2bade9f0ea4a1e93fb08e15e0389242a6e7ec6d6 Mon Sep 17 00:00:00 2001 From: Maximilian Kaul Date: Wed, 25 Sep 2024 14:17:32 +0200 Subject: [PATCH 10/30] rename to random --- cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Name.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Name.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Name.kt index 0606d03cbb..c8cf38536a 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Name.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Name.kt @@ -56,7 +56,7 @@ class Name( * prefixed by [prefix], followed by a separator character [separatorChar] and finalized by * a random UUID ("-" separators also replaced with [separatorChar]). */ - fun getRandomTempName(prefix: String, separatorChar: Char = '_'): Name { + fun random(prefix: String, separatorChar: Char = '_'): Name { val randomPart = Uuid.random().toString().replace('-', separatorChar) return Name(localName = prefix + separatorChar + randomPart) } From be4f90af4e33fe41666a8275fd07c7c1ec95f8c0 Mon Sep 17 00:00:00 2001 From: Maximilian Kaul Date: Wed, 25 Sep 2024 14:27:18 +0200 Subject: [PATCH 11/30] extract constant prefix string --- .../aisec/cpg/frontends/python/PythonHandler.kt | 8 ++++++++ .../aisec/cpg/frontends/python/StatementHandler.kt | 2 +- .../aisec/cpg/frontends/python/PythonFrontendTest.kt | 2 +- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonHandler.kt b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonHandler.kt index f6d1fc2db8..23a48a3327 100644 --- a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonHandler.kt +++ b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonHandler.kt @@ -49,4 +49,12 @@ abstract class PythonHandler( } abstract fun handleNode(node: HandlerNode): ResultNode + + companion object { + /** + * A prefix to add to random names when handling for loops with multiple variables and + * having to add implicit assignments for the unwrapping process. + */ + const val LOOP_VAR_PREFIX = "loopMultiVarHelperVar" + } } diff --git a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt index aac251b81c..da4af6a219 100644 --- a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt +++ b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt @@ -220,7 +220,7 @@ class StatementHandler(frontend: PythonLanguageFrontend) : private fun getDummyAssign( loopVar: InitializerListExpression ): Pair { - val tempVarName = Name.random(prefix = "loopMultiVarHelperVar") + val tempVarName = Name.random(prefix = LOOP_VAR_PREFIX) val tempRef = newReference(name = tempVarName) tempRef.isImplicit = true val assign = diff --git a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt index f211598669..bc6ee19ec7 100644 --- a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt +++ b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt @@ -822,7 +822,7 @@ class PythonFrontendTest : BaseTest() { ) // including one dummy variable introduced for the loop var assertEquals( 4, - p.variables.filter { !it.name.localName.contains("loopMultiVarHelperVar") }.size + p.variables.filter { !it.name.localName.contains(PythonHandler.LOOP_VAR_PREFIX) }.size ) assertEquals(2, p.statements.size) From b19cea9948a775bd98fc7a86a526cb52926815f3 Mon Sep 17 00:00:00 2001 From: Maximilian Kaul Date: Wed, 25 Sep 2024 15:17:26 +0200 Subject: [PATCH 12/30] change declaration placement --- .../aisec/cpg/passes/PythonAddDeclarationsPass.kt | 2 +- .../aisec/cpg/frontends/python/PythonFrontendTest.kt | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/PythonAddDeclarationsPass.kt b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/PythonAddDeclarationsPass.kt index aae0af5991..34132f39c2 100644 --- a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/PythonAddDeclarationsPass.kt +++ b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/PythonAddDeclarationsPass.kt @@ -164,7 +164,7 @@ class PythonAddDeclarationsPass(ctx: TranslationContext) : ComponentPass(ctx) { when (val forVar = node.variable) { is Reference -> { val handled = handleReference(forVar) - (handled as? Declaration)?.let { forVar.addDeclaration(it) } + (handled as? Declaration)?.let { scopeManager.addDeclaration(it) } } } } diff --git a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt index bc6ee19ec7..5f0b6c2610 100644 --- a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt +++ b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt @@ -842,8 +842,11 @@ class PythonFrontendTest : BaseTest() { val forVariable = forStmt.variable as? Reference assertNotNull(forVariable) - val forVarDecl = forVariable.declarations.first() - assertEquals(1, forVariable.declarations.size) + val forVarDecl = + p.declarations.firstOrNull { + it.name.localName.contains((PythonHandler.LOOP_VAR_PREFIX)) + } + assertNotNull(forVarDecl) assertEquals(forVarDecl, forVariable.refersTo) val iter = forStmt.iterable as? Reference From 5a4db8ac069bedb888cb7832c96da576bbcf5b99 Mon Sep 17 00:00:00 2001 From: Maximilian Kaul Date: Wed, 25 Sep 2024 17:14:10 +0200 Subject: [PATCH 13/30] polishing the code --- .../cpg/frontends/python/StatementHandler.kt | 59 +++++++++++++++---- 1 file changed, 46 insertions(+), 13 deletions(-) diff --git a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt index 85b0acd39f..776341adcd 100644 --- a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt +++ b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt @@ -35,6 +35,7 @@ import de.fraunhofer.aisec.cpg.graph.Annotation import de.fraunhofer.aisec.cpg.graph.declarations.* import de.fraunhofer.aisec.cpg.graph.statements.AssertStatement import de.fraunhofer.aisec.cpg.graph.statements.DeclarationStatement +import de.fraunhofer.aisec.cpg.graph.statements.ForEachStatement import de.fraunhofer.aisec.cpg.graph.statements.Statement import de.fraunhofer.aisec.cpg.graph.statements.expressions.AssignExpression import de.fraunhofer.aisec.cpg.graph.statements.expressions.Block @@ -171,7 +172,26 @@ class StatementHandler(frontend: PythonLanguageFrontend) : return ret } - private fun handleFor(node: Python.AST.NormalOrAsyncFor): Statement { + /** + * Translates a Python [`For`](https://docs.python.org/3/library/ast.html#ast.For) into an + * [ForEachStatement]. + * + * PPython supports implicit unpacking of multiple loop variables. To map this to CPG node, we + * translate the following implicit unpacking of code like this: + * ```python + * for a, b, c in someNestedList: + * pass + * ``` + * + * to have only one loop variable and add the unpacking statement to the top of the loop body + * like this: + * ```python + * for tempVar in someNestedList: + * a, b, c = tempVar + * pass + * ``` + */ + private fun handleFor(node: Python.AST.NormalOrAsyncFor): ForEachStatement { val ret = newForEachStatement(rawNode = node) if (node is IsAsync) { ret.addDeclaration( @@ -183,26 +203,31 @@ class StatementHandler(frontend: PythonLanguageFrontend) : } ret.iterable = frontend.expressionHandler.handle(node.iter) - val loopVar = frontend.expressionHandler.handle(node.target) - when (loopVar) { - is InitializerListExpression -> { + when (val loopVar = frontend.expressionHandler.handle(node.target)) { + is InitializerListExpression -> { // unpacking + val (tempVarRef, unpackingAssignment) = getUnpackingNodes(loopVar) - val (dummyVarRef, dummyAssign) = getDummyAssign(loopVar) + ret.variable = tempVarRef - ret.variable = dummyVarRef val body = makeBlock(node.body, parentNode = node) - - body.statements.add(0, dummyAssign) - + body.statements.add( + 0, + unpackingAssignment + ) // add the unpacking instruction to the top of the loop body ret.statement = body } - is Reference -> { + is Reference -> { // only one var ret.variable = loopVar ret.statement = makeBlock(node.body, parentNode = node) } else -> { - TODO() + ret.variable = + newProblemExpression( + problem = "handleFor: cannot handle loop variable.", + rawNode = node.target + ) + ret.statement = makeBlock(node.body, parentNode = node) } } @@ -216,8 +241,16 @@ class StatementHandler(frontend: PythonLanguageFrontend) : return ret } - /** TODO */ - private fun getDummyAssign( + /** + * This function creates two things: + * - A [Reference] to a variable with a random [Name] + * - An [AssignExpression] assigning the reference above to the [loopVar] input + * + * This is used in [handleFor] when loops have multiple loop variables to iterate over with + * automatic unpacking. We translate this implicit unpacking to multiple CPG nodes, as the CPG + * does not support automatic unpacking. + */ + private fun getUnpackingNodes( loopVar: InitializerListExpression ): Pair { val tempVarName = Name.random(prefix = LOOP_VAR_PREFIX) From d07712296bc9c03c544f6e5a41fed744c7763639 Mon Sep 17 00:00:00 2001 From: Maximilian Kaul Date: Wed, 25 Sep 2024 17:24:33 +0200 Subject: [PATCH 14/30] doc++ --- .../de/fraunhofer/aisec/cpg/passes/PythonAddDeclarationsPass.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/PythonAddDeclarationsPass.kt b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/PythonAddDeclarationsPass.kt index 34132f39c2..100791c74a 100644 --- a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/PythonAddDeclarationsPass.kt +++ b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/PythonAddDeclarationsPass.kt @@ -159,7 +159,7 @@ class PythonAddDeclarationsPass(ctx: TranslationContext) : ComponentPass(ctx) { } } - // TODO document why this is necessary and implement for other possible places + // New variables can also be declared as `variable` in a [ForEachStatement] private fun handleForEach(node: ForEachStatement) { when (val forVar = node.variable) { is Reference -> { From ed00ededffce1c6973ea655bd270f907cf131528 Mon Sep 17 00:00:00 2001 From: Christian Banse Date: Wed, 25 Sep 2024 19:13:46 +0200 Subject: [PATCH 15/30] Open a block and function scope for python functions --- .../cpg/frontends/python/StatementHandler.kt | 22 +++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt index d08f253aaf..fdf05aed61 100644 --- a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt +++ b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt @@ -397,7 +397,12 @@ class StatementHandler(frontend: PythonLanguageFrontend) : handleArguments(s.args, result, recordDeclaration) if (s.body.isNotEmpty()) { - result.body = makeBlock(s.body, parentNode = s) + // Make sure we open a new (block) scope for the function body. This is not a 1:1 + // mapping to python scopes, since python only has a "function scope", but in the CPG + // the function scope only comprises the function arguments, and we need a block scope + // to + // hold all local variables within the function body. + result.body = makeBlock(s.body, parentNode = s, enterScope = true) } frontend.scopeManager.leaveScope(result) @@ -603,16 +608,29 @@ class StatementHandler(frontend: PythonLanguageFrontend) : * This function "wraps" a list of [Python.AST.BaseStmt] nodes into a [Block]. Since the list * itself does not have a code/location, we need to employ [codeAndLocationFromChildren] on the * [parentNode]. + * + * Optionally, a new scope will be opened when [enterScope] is specified. This should be done + * VERY carefully, as Python has a very limited set of scopes and is most likely only to be used + * by [handleFunctionDef]. */ private fun makeBlock( stmts: List, - parentNode: Python.AST.WithLocation + parentNode: Python.AST.WithLocation, + enterScope: Boolean = false, ): Block { val result = newBlock() + if (enterScope) { + frontend.scopeManager.enterScope(result) + } + for (stmt in stmts) { result.statements += handle(stmt) } + if (enterScope) { + frontend.scopeManager.leaveScope(result) + } + // Try to retrieve the code and location from the parent node, if it is a base stmt var baseStmt = parentNode as? Python.AST.BaseStmt return if (baseStmt != null) { From a11245909f958bb47eff1874f71414b42c1c7a19 Mon Sep 17 00:00:00 2001 From: Maximilian Kaul Date: Thu, 26 Sep 2024 16:13:19 +0200 Subject: [PATCH 16/30] Update cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt Co-authored-by: Christian Banse --- .../fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt index 9c715e3298..735b2d20ef 100644 --- a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt +++ b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt @@ -246,7 +246,7 @@ class StatementHandler(frontend: PythonLanguageFrontend) : else -> { ret.variable = newProblemExpression( - problem = "handleFor: cannot handle loop variable.", + problem = "handleFor: cannot handle loop variable of type ${loopVar::class.simpleName}.", rawNode = node.target ) ret.statement = makeBlock(node.body, parentNode = node) From fe073f1437b9c191b15bca78e17f475947be4589 Mon Sep 17 00:00:00 2001 From: Maximilian Kaul Date: Thu, 26 Sep 2024 16:13:53 +0200 Subject: [PATCH 17/30] Update cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt Co-authored-by: Christian Banse --- .../fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt index 735b2d20ef..12f956620c 100644 --- a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt +++ b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt @@ -276,7 +276,7 @@ class StatementHandler(frontend: PythonLanguageFrontend) : loopVar: InitializerListExpression ): Pair { val tempVarName = Name.random(prefix = LOOP_VAR_PREFIX) - val tempRef = newReference(name = tempVarName) + val tempRef = newReference(name = tempVarName).implicit().codeAndLocationFrom(loopVar) tempRef.isImplicit = true val assign = newAssignExpression( From d812822d490281e605f5270482fa581563dc5899 Mon Sep 17 00:00:00 2001 From: Maximilian Kaul Date: Thu, 26 Sep 2024 16:14:13 +0200 Subject: [PATCH 18/30] Update cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt Co-authored-by: Christian Banse --- .../fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt index 5f0b6c2610..330119b8b4 100644 --- a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt +++ b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt @@ -889,7 +889,7 @@ class PythonFrontendTest : BaseTest() { assertEquals(t3Decl, t3RefAssign.refersTo) // print("bug ... {} {} {}".format(t1, t2, t3)) - val forBodyStmt = forBody.statements[1] as? CallExpression + val forBodyStmt = forBody.statements(1) assertNotNull(forBodyStmt) assertLocalName("print", forBodyStmt) From 138ce62686560d5ab1d64d110d3c063d9cc09900 Mon Sep 17 00:00:00 2001 From: Maximilian Kaul Date: Thu, 26 Sep 2024 16:21:39 +0200 Subject: [PATCH 19/30] code review --- .../cpg/frontends/python/StatementHandler.kt | 5 +- .../frontends/python/PythonFrontendTest.kt | 66 ++++++++++--------- 2 files changed, 38 insertions(+), 33 deletions(-) diff --git a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt index 12f956620c..14f3cb5a16 100644 --- a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt +++ b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt @@ -198,7 +198,7 @@ class StatementHandler(frontend: PythonLanguageFrontend) : * Translates a Python [`For`](https://docs.python.org/3/library/ast.html#ast.For) into an * [ForEachStatement]. * - * PPython supports implicit unpacking of multiple loop variables. To map this to CPG node, we + * Python supports implicit unpacking of multiple loop variables. To map this to CPG node, we * translate the following implicit unpacking of code like this: * ```python * for a, b, c in someNestedList: @@ -246,7 +246,8 @@ class StatementHandler(frontend: PythonLanguageFrontend) : else -> { ret.variable = newProblemExpression( - problem = "handleFor: cannot handle loop variable of type ${loopVar::class.simpleName}.", + problem = + "handleFor: cannot handle loop variable of type ${loopVar::class.simpleName}.", rawNode = node.target ) ret.statement = makeBlock(node.body, parentNode = node) diff --git a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt index 330119b8b4..46e7d208fc 100644 --- a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt +++ b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt @@ -104,7 +104,7 @@ class PythonFrontendTest : BaseTest() { val p = tu.namespaces["function"] assertNotNull(p) - val foo = p.declarations.first() as? FunctionDeclaration + val foo = p.declarations.firstOrNull() as? FunctionDeclaration assertNotNull(foo) val bar = p.declarations[1] as? FunctionDeclaration @@ -115,13 +115,13 @@ class PythonFrontendTest : BaseTest() { assertNotNull(callExpression) assertLocalName("bar", callExpression) - assertEquals(bar, callExpression.invokes.first()) + assertEquals(bar, callExpression.invokes.firstOrNull()) val edge = callExpression.argumentEdges[1] assertNotNull(edge) assertEquals("s2", edge.name) - val s = bar.parameters.first() + val s = bar.parameters.firstOrNull() assertNotNull(s) assertLocalName("s", s) assertEquals(tu.primitiveType("str"), s.type) @@ -137,7 +137,7 @@ class PythonFrontendTest : BaseTest() { assertFullName("print", callExpression) - val literal = callExpression.arguments.first() as? Literal<*> + val literal = callExpression.arguments.firstOrNull() as? Literal<*> assertNotNull(literal) assertEquals("bar(s) here: ", literal.value) @@ -152,7 +152,7 @@ class PythonFrontendTest : BaseTest() { val stmt = compStmt.statements[1] as? AssignExpression assertNotNull(stmt) - val a = stmt.declarations.first() as? VariableDeclaration + val a = stmt.declarations.firstOrNull() as? VariableDeclaration assertNotNull(a) assertLocalName("a", a) @@ -200,7 +200,7 @@ class PythonFrontendTest : BaseTest() { val body = main.body as? Block assertNotNull(body) - val sel = (body.statements.first() as? AssignExpression)?.declarations?.first() + val sel = (body.statements.firstOrNull() as? AssignExpression)?.declarations?.firstOrNull() assertNotNull(sel) assertLocalName("sel", sel) assertEquals(tu.primitiveType("bool"), sel.type) @@ -239,7 +239,7 @@ class PythonFrontendTest : BaseTest() { assertLocalName("SomeClass", cls) assertEquals(1, cls.methods.size) - val clsfunc = cls.methods.first() + val clsfunc = cls.methods.firstOrNull() assertLocalName("someFunc", clsfunc) assertLocalName("foo", foo) @@ -253,17 +253,17 @@ class PythonFrontendTest : BaseTest() { val s2 = body.statements[1] as? MemberCallExpression assertNotNull(s2) - val c1 = s1.declarations.first() as? VariableDeclaration + val c1 = s1.declarations.firstOrNull() as? VariableDeclaration assertNotNull(c1) assertLocalName("c1", c1) val ctor = c1.firstAssignment as? ConstructExpression assertNotNull(ctor) - assertEquals(ctor.constructor, cls.constructors.first() as? ConstructorDeclaration) + assertEquals(ctor.constructor, cls.constructors.firstOrNull() as? ConstructorDeclaration) assertFullName("simple_class.SomeClass", c1.type) assertEquals(c1, (s2.base as? Reference)?.refersTo) assertEquals(1, s2.invokes.size) - assertEquals(clsfunc, s2.invokes.first()) + assertEquals(clsfunc, s2.invokes.firstOrNull()) // member } @@ -281,7 +281,7 @@ class PythonFrontendTest : BaseTest() { val main = p.functions["foo"] assertNotNull(main) - val assignExpr = (main.body as? Block)?.statements?.first() as? AssignExpression + val assignExpr = (main.body as? Block)?.statements?.firstOrNull() as? AssignExpression assertNotNull(assignExpr) val foo = assignExpr.declarations.firstOrNull() as? VariableDeclaration @@ -419,8 +419,9 @@ class PythonFrontendTest : BaseTest() { // self.somevar = i val someVarDeclaration = - ((bar.body as? Block)?.statements?.get(0) as? AssignExpression)?.declarations?.first() - as? FieldDeclaration + ((bar.body as? Block)?.statements?.get(0) as? AssignExpression) + ?.declarations + ?.firstOrNull() as? FieldDeclaration assertNotNull(someVarDeclaration) assertLocalName("somevar", someVarDeclaration) assertEquals(i, (someVarDeclaration.firstAssignment as? Reference)?.refersTo) @@ -567,13 +568,16 @@ class PythonFrontendTest : BaseTest() { val ifThen = (countStmt.thenStatement as? Block)?.statements?.get(0) as? CallExpression assertNotNull(ifThen) - assertEquals(methCount, ifThen.invokes.first()) - assertEquals(countParam, (ifThen.arguments.first() as? Reference)?.refersTo) + assertEquals(methCount, ifThen.invokes.firstOrNull()) + assertEquals(countParam, (ifThen.arguments.firstOrNull() as? Reference)?.refersTo) assertNull(countStmt.elseStatement) // class c1(counter) assertLocalName("c1", clsC1) - assertEquals(clsCounter, (clsC1.superClasses.first() as? ObjectType)?.recordDeclaration) + assertEquals( + clsCounter, + (clsC1.superClasses.firstOrNull() as? ObjectType)?.recordDeclaration + ) assertEquals(1, clsC1.fields.size) val field = clsC1.fields[0] @@ -872,10 +876,10 @@ class PythonFrontendTest : BaseTest() { rest of the loop ``` */ - val forVariableImplicitStmt = forBody.statements.first() as? AssignExpression + val forVariableImplicitStmt = forBody.statements.firstOrNull() as? AssignExpression assertNotNull(forVariableImplicitStmt) assertEquals("=", forVariableImplicitStmt.operatorCode) - assertEquals(forStmt.variable, forVariableImplicitStmt.rhs.first()) + assertEquals(forStmt.variable, forVariableImplicitStmt.rhs.firstOrNull()) val (t1Decl, t2Decl, t3Decl) = forVariableImplicitStmt.declarations val (t1RefAssign, t2RefAssign, t3RefAssign) = forVariableImplicitStmt.lhs assertNotNull(t1Decl) @@ -884,9 +888,9 @@ class PythonFrontendTest : BaseTest() { assertNotNull(t1RefAssign as? Reference) assertNotNull(t2RefAssign as? Reference) assertNotNull(t3RefAssign as? Reference) - assertEquals(t1Decl, t1RefAssign.refersTo) - assertEquals(t2Decl, t2RefAssign.refersTo) - assertEquals(t3Decl, t3RefAssign.refersTo) + assertRefersTo(t1RefAssign, t1Decl) + assertRefersTo(t2RefAssign, t2Decl) + assertRefersTo(t3RefAssign, t3Decl) // print("bug ... {} {} {}".format(t1, t2, t3)) val forBodyStmt = forBody.statements(1) @@ -897,13 +901,13 @@ class PythonFrontendTest : BaseTest() { assertNotNull(printArg) val formatArgT1 = printArg.arguments[0] as? Reference assertNotNull(formatArgT1) - assertEquals(t1Decl, formatArgT1.refersTo) + assertRefersTo(formatArgT1, t1Decl) val formatArgT2 = printArg.arguments[1] as? Reference assertNotNull(formatArgT2) - assertEquals(t2Decl, formatArgT2.refersTo) + assertRefersTo(formatArgT2, t2Decl) val formatArgT3 = printArg.arguments[2] as? Reference assertNotNull(formatArgT3) - assertEquals(t3Decl, formatArgT3.refersTo) + assertRefersTo(formatArgT3, t3Decl) } @Test @@ -971,12 +975,12 @@ class PythonFrontendTest : BaseTest() { assertEquals(1, functions.size) assertEquals( "# a function", - functions.first().comment, + functions.firstOrNull()?.comment, ) val literals = commentedNodes.filterIsInstance>() assertEquals(1, literals.size) - assertEquals("# comment start", literals.first().comment) + assertEquals("# comment start", literals.firstOrNull()?.comment) val params = commentedNodes.filterIsInstance() assertEquals(2, params.size) @@ -985,12 +989,12 @@ class PythonFrontendTest : BaseTest() { val assignment = commentedNodes.filterIsInstance() assertEquals(2, assignment.size) - assertEquals("# A comment# a number", assignment.first().comment) + assertEquals("# A comment# a number", assignment.firstOrNull()?.comment) assertEquals("# comment end", assignment.last().comment) val block = commentedNodes.filterIsInstance() assertEquals(1, block.size) - assertEquals("# foo", block.first().comment) + assertEquals("# foo", block.firstOrNull()?.comment) val kvs = commentedNodes.filterIsInstance() assertEquals(2, kvs.size) @@ -1069,10 +1073,10 @@ class PythonFrontendTest : BaseTest() { // dataflow from first loop to foo call val loopVar = firstLoop.variable as? Reference assertNotNull(loopVar) - assert(fooCall.arguments.first().prevDFG.contains(loopVar)) + assertTrue(fooCall.arguments.firstOrNull()?.prevDFG?.contains(loopVar) == true) // dataflow from var declaration to foo call (in case for loop is not executed) - assert(fooCall.arguments.first().prevDFG.contains(varDefinedBeforeLoopRef)) + assert(fooCall.arguments.firstOrNull()?.prevDFG?.contains(varDefinedBeforeLoopRef) == true) // dataflow from range call to loop variable val secondLoopIterable = secondLoop.iterable as? CallExpression @@ -1084,7 +1088,7 @@ class PythonFrontendTest : BaseTest() { // dataflow from second loop var to bar call assertEquals( (secondLoop.variable as? Reference), - barCall.arguments.first().prevDFG.firstOrNull() + barCall.arguments.firstOrNull()?.prevDFG?.firstOrNull() ) } From 090a617dd8a991aca04fef85535fe23dd8a7b8ea Mon Sep 17 00:00:00 2001 From: Maximilian Kaul Date: Thu, 26 Sep 2024 16:24:52 +0200 Subject: [PATCH 20/30] fixed code quality problems --- .../cpg/frontends/python/PythonFrontendTest.kt | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt index 46e7d208fc..fa825d2fb5 100644 --- a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt +++ b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt @@ -152,7 +152,7 @@ class PythonFrontendTest : BaseTest() { val stmt = compStmt.statements[1] as? AssignExpression assertNotNull(stmt) - val a = stmt.declarations.firstOrNull() as? VariableDeclaration + val a = stmt.declarations.firstOrNull() assertNotNull(a) assertLocalName("a", a) @@ -253,12 +253,12 @@ class PythonFrontendTest : BaseTest() { val s2 = body.statements[1] as? MemberCallExpression assertNotNull(s2) - val c1 = s1.declarations.firstOrNull() as? VariableDeclaration + val c1 = s1.declarations.firstOrNull() assertNotNull(c1) assertLocalName("c1", c1) val ctor = c1.firstAssignment as? ConstructExpression assertNotNull(ctor) - assertEquals(ctor.constructor, cls.constructors.firstOrNull() as? ConstructorDeclaration) + assertEquals(ctor.constructor, cls.constructors.firstOrNull()) assertFullName("simple_class.SomeClass", c1.type) assertEquals(c1, (s2.base as? Reference)?.refersTo) @@ -284,7 +284,7 @@ class PythonFrontendTest : BaseTest() { val assignExpr = (main.body as? Block)?.statements?.firstOrNull() as? AssignExpression assertNotNull(assignExpr) - val foo = assignExpr.declarations.firstOrNull() as? VariableDeclaration + val foo = assignExpr.declarations.firstOrNull() assertNotNull(foo) assertLocalName("foo", foo) assertEquals(tu.primitiveType("int"), foo.type) @@ -490,9 +490,9 @@ class PythonFrontendTest : BaseTest() { assertEquals(1, recordFoo.methods.size) assertEquals(1, recordFoo.constructors.size) - val fooCtor = recordFoo.constructors[0] as? ConstructorDeclaration + val fooCtor = recordFoo.constructors[0] assertNotNull(fooCtor) - val foobar = recordFoo.methods[0] as? MethodDeclaration + val foobar = recordFoo.methods[0] assertNotNull(foobar) assertLocalName("__init__", fooCtor) @@ -788,7 +788,7 @@ class PythonFrontendTest : BaseTest() { val main = p.functions["main"] assertNotNull(main) - val mainBody = (main as? FunctionDeclaration)?.body as? Block + val mainBody = main.body as? Block assertNotNull(mainBody) val whlStmt = mainBody.statements[3] as? WhileStatement @@ -998,8 +998,8 @@ class PythonFrontendTest : BaseTest() { val kvs = commentedNodes.filterIsInstance() assertEquals(2, kvs.size) - assertEquals("# a entry", kvs.first { it.code?.contains("a") ?: false }.comment) - assertEquals("# b entry", kvs.first { it.code?.contains("b") ?: false }.comment) + assertEquals("# a entry", kvs.first { it.code?.contains("a") == true }.comment) + assertEquals("# b entry", kvs.first { it.code?.contains("b") == true }.comment) } @Test From 64f37968629bd92f84711b4b594adcc3937cc81c Mon Sep 17 00:00:00 2001 From: Maximilian Kaul Date: Thu, 26 Sep 2024 16:21:39 +0200 Subject: [PATCH 21/30] Replace first() with firstOrNull() --- .../frontends/python/PythonFrontendTest.kt | 50 ++++++++++--------- 1 file changed, 27 insertions(+), 23 deletions(-) diff --git a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt index 5d63e06d6d..3746a764c3 100644 --- a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt +++ b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt @@ -104,7 +104,7 @@ class PythonFrontendTest : BaseTest() { val p = tu.namespaces["function"] assertNotNull(p) - val foo = p.declarations.first() as? FunctionDeclaration + val foo = p.declarations.firstOrNull() as? FunctionDeclaration assertNotNull(foo) val bar = p.declarations[1] as? FunctionDeclaration @@ -115,13 +115,13 @@ class PythonFrontendTest : BaseTest() { assertNotNull(callExpression) assertLocalName("bar", callExpression) - assertEquals(bar, callExpression.invokes.first()) + assertEquals(bar, callExpression.invokes.firstOrNull()) val edge = callExpression.argumentEdges[1] assertNotNull(edge) assertEquals("s2", edge.name) - val s = bar.parameters.first() + val s = bar.parameters.firstOrNull() assertNotNull(s) assertLocalName("s", s) assertEquals(tu.primitiveType("str"), s.type) @@ -137,7 +137,7 @@ class PythonFrontendTest : BaseTest() { assertFullName("print", callExpression) - val literal = callExpression.arguments.first() as? Literal<*> + val literal = callExpression.arguments.firstOrNull() as? Literal<*> assertNotNull(literal) assertEquals("bar(s) here: ", literal.value) @@ -152,7 +152,7 @@ class PythonFrontendTest : BaseTest() { val stmt = compStmt.statements[1] as? AssignExpression assertNotNull(stmt) - val a = stmt.declarations.first() as? VariableDeclaration + val a = stmt.declarations.firstOrNull() as? VariableDeclaration assertNotNull(a) assertLocalName("a", a) @@ -200,7 +200,7 @@ class PythonFrontendTest : BaseTest() { val body = main.body as? Block assertNotNull(body) - val sel = (body.statements.first() as? AssignExpression)?.declarations?.first() + val sel = (body.statements.firstOrNull() as? AssignExpression)?.declarations?.firstOrNull() assertNotNull(sel) assertLocalName("sel", sel) assertEquals(tu.primitiveType("bool"), sel.type) @@ -239,7 +239,7 @@ class PythonFrontendTest : BaseTest() { assertLocalName("SomeClass", cls) assertEquals(1, cls.methods.size) - val clsfunc = cls.methods.first() + val clsfunc = cls.methods.firstOrNull() assertLocalName("someFunc", clsfunc) assertLocalName("foo", foo) @@ -253,17 +253,17 @@ class PythonFrontendTest : BaseTest() { val s2 = body.statements[1] as? MemberCallExpression assertNotNull(s2) - val c1 = s1.declarations.first() as? VariableDeclaration + val c1 = s1.declarations.firstOrNull() as? VariableDeclaration assertNotNull(c1) assertLocalName("c1", c1) val ctor = c1.firstAssignment as? ConstructExpression assertNotNull(ctor) - assertEquals(ctor.constructor, cls.constructors.first() as? ConstructorDeclaration) + assertEquals(ctor.constructor, cls.constructors.firstOrNull() as? ConstructorDeclaration) assertFullName("simple_class.SomeClass", c1.type) assertEquals(c1, (s2.base as? Reference)?.refersTo) assertEquals(1, s2.invokes.size) - assertEquals(clsfunc, s2.invokes.first()) + assertEquals(clsfunc, s2.invokes.firstOrNull()) // member } @@ -281,7 +281,7 @@ class PythonFrontendTest : BaseTest() { val main = p.functions["foo"] assertNotNull(main) - val assignExpr = (main.body as? Block)?.statements?.first() as? AssignExpression + val assignExpr = (main.body as? Block)?.statements?.firstOrNull() as? AssignExpression assertNotNull(assignExpr) val foo = assignExpr.declarations.firstOrNull() as? VariableDeclaration @@ -419,8 +419,9 @@ class PythonFrontendTest : BaseTest() { // self.somevar = i val someVarDeclaration = - ((bar.body as? Block)?.statements?.get(0) as? AssignExpression)?.declarations?.first() - as? FieldDeclaration + ((bar.body as? Block)?.statements?.get(0) as? AssignExpression) + ?.declarations + ?.firstOrNull() as? FieldDeclaration assertNotNull(someVarDeclaration) assertLocalName("somevar", someVarDeclaration) assertEquals(i, (someVarDeclaration.firstAssignment as? Reference)?.refersTo) @@ -567,13 +568,16 @@ class PythonFrontendTest : BaseTest() { val ifThen = (countStmt.thenStatement as? Block)?.statements?.get(0) as? CallExpression assertNotNull(ifThen) - assertEquals(methCount, ifThen.invokes.first()) - assertEquals(countParam, (ifThen.arguments.first() as? Reference)?.refersTo) + assertEquals(methCount, ifThen.invokes.firstOrNull()) + assertEquals(countParam, (ifThen.arguments.firstOrNull() as? Reference)?.refersTo) assertNull(countStmt.elseStatement) // class c1(counter) assertLocalName("c1", clsC1) - assertEquals(clsCounter, (clsC1.superClasses.first() as? ObjectType)?.recordDeclaration) + assertEquals( + clsCounter, + (clsC1.superClasses.firstOrNull() as? ObjectType)?.recordDeclaration + ) assertEquals(1, clsC1.fields.size) val field = clsC1.fields[0] @@ -937,12 +941,12 @@ class PythonFrontendTest : BaseTest() { assertEquals(1, functions.size) assertEquals( "# a function", - functions.first().comment, + functions.firstOrNull()?.comment, ) val literals = commentedNodes.filterIsInstance>() assertEquals(1, literals.size) - assertEquals("# comment start", literals.first().comment) + assertEquals("# comment start", literals.firstOrNull()?.comment) val params = commentedNodes.filterIsInstance() assertEquals(2, params.size) @@ -951,12 +955,12 @@ class PythonFrontendTest : BaseTest() { val assignment = commentedNodes.filterIsInstance() assertEquals(2, assignment.size) - assertEquals("# A comment# a number", assignment.first().comment) + assertEquals("# A comment# a number", assignment.firstOrNull()?.comment) assertEquals("# comment end", assignment.last().comment) val block = commentedNodes.filterIsInstance() assertEquals(1, block.size) - assertEquals("# foo", block.first().comment) + assertEquals("# foo", block.firstOrNull()?.comment) val kvs = commentedNodes.filterIsInstance() assertEquals(2, kvs.size) @@ -1035,10 +1039,10 @@ class PythonFrontendTest : BaseTest() { // dataflow from first loop to foo call val loopVar = firstLoop.variable as? Reference assertNotNull(loopVar) - assert(fooCall.arguments.first().prevDFG.contains(loopVar)) + assertTrue(fooCall.arguments.firstOrNull()?.prevDFG?.contains(loopVar) == true) // dataflow from var declaration to foo call (in case for loop is not executed) - assert(fooCall.arguments.first().prevDFG.contains(varDefinedBeforeLoopRef)) + assert(fooCall.arguments.firstOrNull()?.prevDFG?.contains(varDefinedBeforeLoopRef) == true) // dataflow from range call to loop variable val secondLoopIterable = secondLoop.iterable as? CallExpression @@ -1050,7 +1054,7 @@ class PythonFrontendTest : BaseTest() { // dataflow from second loop var to bar call assertEquals( (secondLoop.variable as? Reference), - barCall.arguments.first().prevDFG.firstOrNull() + barCall.arguments.firstOrNull()?.prevDFG?.firstOrNull() ) } From b63975335036cf7a969c2b65b31a883e1e00d03e Mon Sep 17 00:00:00 2001 From: Maximilian Kaul Date: Fri, 27 Sep 2024 10:03:52 +0200 Subject: [PATCH 22/30] merge error --- .../fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt index b729a16644..9eef5f0c1e 100644 --- a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt +++ b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt @@ -855,7 +855,7 @@ class PythonFrontendTest : BaseTest() { val iter = forStmt.iterable as? Reference assertNotNull(iter) - assertEquals(testDeclaration, iter.refersTo) + assertRefersTo(iter, testDeclaration) val forBody = forStmt.statement assertIs(forBody) From a5b6a6c9ecd9c436580918805394b8915bceec17 Mon Sep 17 00:00:00 2001 From: Maximilian Kaul Date: Fri, 27 Sep 2024 10:05:06 +0200 Subject: [PATCH 23/30] merge error --- .../aisec/cpg/frontends/python/PythonFrontendTest.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt index 9eef5f0c1e..d4349032ba 100644 --- a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt +++ b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt @@ -853,8 +853,8 @@ class PythonFrontendTest : BaseTest() { assertNotNull(forVarDecl) assertRefersTo(forVariable, forVarDecl) - val iter = forStmt.iterable as? Reference - assertNotNull(iter) + val iter = forStmt.iterable + assertIs(iter) assertRefersTo(iter, testDeclaration) val forBody = forStmt.statement From db89326b12ea2b3853b443da92350a70182e99c3 Mon Sep 17 00:00:00 2001 From: Maximilian Kaul Date: Fri, 27 Sep 2024 12:20:33 +0200 Subject: [PATCH 24/30] assertIs --- .../frontends/python/ExpressionHandlerTest.kt | 37 +- .../frontends/python/PythonFrontendTest.kt | 339 +++++++++--------- .../statementHandler/StatementHandlerTest.kt | 9 +- 3 files changed, 191 insertions(+), 194 deletions(-) diff --git a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandlerTest.kt b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandlerTest.kt index 1d0be6d624..51da0d504d 100644 --- a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandlerTest.kt +++ b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandlerTest.kt @@ -43,9 +43,8 @@ class ExpressionHandlerTest { } assertNotNull(result) - val twoBoolOpCondition = - result.functions["twoBoolOp"]?.ifs?.singleOrNull()?.condition as? BinaryOperator - assertNotNull(twoBoolOpCondition) + val twoBoolOpCondition = result.functions["twoBoolOp"]?.ifs?.singleOrNull()?.condition + assertIs(twoBoolOpCondition) assertEquals("and", twoBoolOpCondition.operatorCode) assertLocalName("a", twoBoolOpCondition.lhs) assertLiteralValue(true, twoBoolOpCondition.rhs) @@ -53,24 +52,23 @@ class ExpressionHandlerTest { // We expect that lhs comes first in the EOG and then the rhs. assertContains(twoBoolOpCondition.lhs.nextEOG, twoBoolOpCondition.rhs) - val threeBoolOpCondition = - result.functions["threeBoolOp"]?.ifs?.singleOrNull()?.condition as? BinaryOperator - assertNotNull(threeBoolOpCondition) + val threeBoolOpCondition = result.functions["threeBoolOp"]?.ifs?.singleOrNull()?.condition + assertIs(threeBoolOpCondition) assertEquals("and", threeBoolOpCondition.operatorCode) assertLocalName("a", threeBoolOpCondition.lhs) - val threeBoolOpConditionRhs = threeBoolOpCondition.rhs as? BinaryOperator - assertNotNull(threeBoolOpConditionRhs) + val threeBoolOpConditionRhs = threeBoolOpCondition.rhs + assertIs(threeBoolOpConditionRhs) assertEquals("and", threeBoolOpConditionRhs.operatorCode) assertLiteralValue(true, threeBoolOpConditionRhs.lhs) assertLocalName("b", threeBoolOpConditionRhs.rhs) val threeBoolOpNoBoolCondition = - result.functions["threeBoolOpNoBool"]?.ifs?.singleOrNull()?.condition as? BinaryOperator - assertNotNull(threeBoolOpNoBoolCondition) + result.functions["threeBoolOpNoBool"]?.ifs?.singleOrNull()?.condition + assertIs(threeBoolOpNoBoolCondition) assertEquals("and", threeBoolOpNoBoolCondition.operatorCode) assertLocalName("a", threeBoolOpNoBoolCondition.lhs) - val threeBoolOpNoBoolConditionRhs = threeBoolOpNoBoolCondition.rhs as? BinaryOperator - assertNotNull(threeBoolOpNoBoolConditionRhs) + val threeBoolOpNoBoolConditionRhs = threeBoolOpNoBoolCondition.rhs + assertIs(threeBoolOpNoBoolConditionRhs) assertEquals("and", threeBoolOpNoBoolConditionRhs.operatorCode) assertLiteralValue(true, threeBoolOpNoBoolConditionRhs.lhs) assertLiteralValue("foo", threeBoolOpNoBoolConditionRhs.rhs) @@ -82,12 +80,12 @@ class ExpressionHandlerTest { val nestedBoolOpDifferentOp = result.functions["nestedBoolOpDifferentOp"]?.ifs?.singleOrNull()?.condition - as? BinaryOperator - assertNotNull(nestedBoolOpDifferentOp) + + assertIs(nestedBoolOpDifferentOp) assertEquals("or", nestedBoolOpDifferentOp.operatorCode) assertLocalName("b", nestedBoolOpDifferentOp.rhs) - val nestedBoolOpDifferentOpLhs = nestedBoolOpDifferentOp.lhs as? BinaryOperator - assertNotNull(nestedBoolOpDifferentOpLhs) + val nestedBoolOpDifferentOpLhs = nestedBoolOpDifferentOp.lhs + assertIs(nestedBoolOpDifferentOpLhs) assertEquals("and", nestedBoolOpDifferentOpLhs.operatorCode) assertLiteralValue(true, nestedBoolOpDifferentOpLhs.rhs) assertLocalName("a", nestedBoolOpDifferentOpLhs.lhs) @@ -100,12 +98,11 @@ class ExpressionHandlerTest { val nestedBoolOpDifferentOp2 = result.functions["nestedBoolOpDifferentOp2"]?.ifs?.singleOrNull()?.condition - as? BinaryOperator - assertNotNull(nestedBoolOpDifferentOp2) + assertIs(nestedBoolOpDifferentOp2) assertEquals("or", nestedBoolOpDifferentOp2.operatorCode) assertLocalName("a", nestedBoolOpDifferentOp2.lhs) - val nestedBoolOpDifferentOp2Rhs = nestedBoolOpDifferentOp2.rhs as? BinaryOperator - assertNotNull(nestedBoolOpDifferentOp2Rhs) + val nestedBoolOpDifferentOp2Rhs = nestedBoolOpDifferentOp2.rhs + assertIs(nestedBoolOpDifferentOp2Rhs) assertEquals("and", nestedBoolOpDifferentOp2Rhs.operatorCode) assertLiteralValue(true, nestedBoolOpDifferentOp2Rhs.lhs) assertLocalName("b", nestedBoolOpDifferentOp2Rhs.rhs) diff --git a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt index d4349032ba..ab3e0c736f 100644 --- a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt +++ b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt @@ -104,15 +104,17 @@ class PythonFrontendTest : BaseTest() { val p = tu.namespaces["function"] assertNotNull(p) - val foo = p.declarations.firstOrNull() as? FunctionDeclaration - assertNotNull(foo) + val foo = p.declarations.firstOrNull() + assertIs(foo) - val bar = p.declarations[1] as? FunctionDeclaration - assertNotNull(bar) + val bar = p.declarations[1] + assertIs(bar) assertEquals(2, bar.parameters.size) - var callExpression = (foo.body as? Block)?.statements?.get(0) as? CallExpression - assertNotNull(callExpression) + val fooBody = foo.body + assertIs(fooBody) + var callExpression = fooBody.statements[0] + assertIs(callExpression) assertLocalName("bar", callExpression) assertEquals(bar, callExpression.invokes.firstOrNull()) @@ -128,52 +130,52 @@ class PythonFrontendTest : BaseTest() { assertLocalName("bar", bar) - val compStmt = bar.body as? Block - assertNotNull(compStmt) + val compStmt = bar.body + assertIs(compStmt) assertNotNull(compStmt.statements) - callExpression = compStmt.statements[0] as? CallExpression - assertNotNull(callExpression) + callExpression = compStmt.statements[0] + assertIs(callExpression) assertFullName("print", callExpression) - val literal = callExpression.arguments.firstOrNull() as? Literal<*> - assertNotNull(literal) + val literal = callExpression.arguments.firstOrNull() + assertIs>(literal) assertEquals("bar(s) here: ", literal.value) assertEquals(tu.primitiveType("str"), literal.type) - val ref = callExpression.arguments[1] as? Reference - assertNotNull(ref) + val ref = callExpression.arguments[1] + assertIs(ref) assertLocalName("s", ref) assertEquals(s, ref.refersTo) - val stmt = compStmt.statements[1] as? AssignExpression - assertNotNull(stmt) + val stmt = compStmt.statements[1] + assertIs(stmt) val a = stmt.declarations.firstOrNull() assertNotNull(a) assertLocalName("a", a) - val op = a.firstAssignment as? BinaryOperator - assertNotNull(op) + val op = a.firstAssignment + assertIs(op) assertEquals("+", op.operatorCode) - val lhs = op.lhs as? Literal<*> - assertNotNull(lhs) + val lhs = op.lhs + assertIs>(lhs) assertEquals(1, (lhs.value as? Long)?.toInt()) - val rhs = op.rhs as? Literal<*> - assertNotNull(rhs) + val rhs = op.rhs + assertIs>(rhs) assertEquals(2, (rhs.value as? Long)?.toInt()) - val r = compStmt.statements[3] as? ReturnStatement - assertNotNull(r) + val r = compStmt.statements[3] + assertIs(r) val s3 = tu.variables["s3"] assertNotNull(s3) @@ -197,21 +199,21 @@ class PythonFrontendTest : BaseTest() { val main = p.functions["foo"] assertNotNull(main) - val body = main.body as? Block - assertNotNull(body) + val body = main.body + assertIs(body) val sel = (body.statements.firstOrNull() as? AssignExpression)?.declarations?.firstOrNull() assertNotNull(sel) assertLocalName("sel", sel) assertEquals(tu.primitiveType("bool"), sel.type) - val firstAssignment = sel.firstAssignment as? Literal<*> - assertNotNull(firstAssignment) + val firstAssignment = sel.firstAssignment + assertIs>(firstAssignment) assertEquals(tu.primitiveType("bool"), firstAssignment.type) assertEquals("True", firstAssignment.code) - val `if` = body.statements[1] as? IfStatement - assertNotNull(`if`) + val `if` = body.statements[1] + assertIs(`if`) } @Test @@ -243,21 +245,21 @@ class PythonFrontendTest : BaseTest() { assertLocalName("someFunc", clsfunc) assertLocalName("foo", foo) - val body = foo.body as? Block - assertNotNull(body) + val body = foo.body + assertIs(body) assertNotNull(body.statements) assertEquals(2, body.statements.size) - val s1 = body.statements[0] as? AssignExpression - assertNotNull(s1) - val s2 = body.statements[1] as? MemberCallExpression - assertNotNull(s2) + val s1 = body.statements[0] + assertIs(s1) + val s2 = body.statements[1] + assertIs(s2) val c1 = s1.declarations.firstOrNull() assertNotNull(c1) assertLocalName("c1", c1) - val ctor = c1.firstAssignment as? ConstructExpression - assertNotNull(ctor) + val ctor = c1.firstAssignment + assertIs(ctor) assertEquals(ctor.constructor, cls.constructors.firstOrNull()) assertFullName("simple_class.SomeClass", c1.type) @@ -281,24 +283,24 @@ class PythonFrontendTest : BaseTest() { val main = p.functions["foo"] assertNotNull(main) - val assignExpr = (main.body as? Block)?.statements?.firstOrNull() as? AssignExpression - assertNotNull(assignExpr) + val assignExpr = (main.body as? Block)?.statements?.firstOrNull() + assertIs(assignExpr) val foo = assignExpr.declarations.firstOrNull() assertNotNull(foo) assertLocalName("foo", foo) assertEquals(tu.primitiveType("int"), foo.type) - val initializer = foo.firstAssignment as? ConditionalExpression - assertNotNull(initializer) + val initializer = foo.firstAssignment + assertIs(initializer) assertEquals(tu.primitiveType("int"), initializer.type) - val ifCond = initializer.condition as? Literal<*> - assertNotNull(ifCond) - val thenExpr = initializer.thenExpression as? Literal<*> - assertNotNull(thenExpr) - val elseExpr = initializer.elseExpression as? Literal<*> - assertNotNull(elseExpr) + val ifCond = initializer.condition + assertIs>(ifCond) + val thenExpr = initializer.thenExpression + assertIs>(thenExpr) + val elseExpr = initializer.elseExpression + assertIs>(elseExpr) assertEquals(tu.primitiveType("bool"), ifCond.type) assertEquals(false, ifCond.value) @@ -360,14 +362,14 @@ class PythonFrontendTest : BaseTest() { assertNotNull(methBar) assertLocalName("bar", methBar) - val barZ = (methBar.body as? Block)?.statements?.get(0) as? MemberExpression - assertNotNull(barZ) + val barZ = (methBar.body as? Block)?.statements?.get(0) + assertIs(barZ) assertEquals(fieldZ, barZ.refersTo) - val barBaz = (methBar.body as? Block)?.statements?.get(1) as? AssignExpression - assertNotNull(barBaz) - val barBazInner = barBaz.declarations[0] as? FieldDeclaration - assertNotNull(barBazInner) + val barBaz = (methBar.body as? Block)?.statements?.get(1) + assertIs(barBaz) + val barBazInner = barBaz.declarations[0] + assertIs(barBazInner) assertLocalName("baz", barBazInner) assertNotNull(barBazInner.firstAssignment) } @@ -421,16 +423,16 @@ class PythonFrontendTest : BaseTest() { val someVarDeclaration = ((bar.body as? Block)?.statements?.get(0) as? AssignExpression) ?.declarations - ?.firstOrNull() as? FieldDeclaration - assertNotNull(someVarDeclaration) + ?.firstOrNull() + assertIs(someVarDeclaration) assertLocalName("somevar", someVarDeclaration) assertEquals(i, (someVarDeclaration.firstAssignment as? Reference)?.refersTo) - val fooMemCall = (foo.body as? Block)?.statements?.get(0) as? MemberCallExpression - assertNotNull(fooMemCall) + val fooMemCall = (foo.body as? Block)?.statements?.get(0) + assertIs(fooMemCall) - val mem = fooMemCall.callee as? MemberExpression - assertNotNull(mem) + val mem = fooMemCall.callee + assertIs(mem) assertLocalName("bar", mem) assertEquals(".", fooMemCall.operatorCode) assertFullName("class_self.Foo.bar", fooMemCall) @@ -503,10 +505,10 @@ class PythonFrontendTest : BaseTest() { assertLocalName("bar", bar) assertEquals(2, (bar.body as? Block)?.statements?.size) - val line1 = (bar.body as? Block)?.statements?.get(0) as? AssignExpression - assertNotNull(line1) - val line2 = (bar.body as? Block)?.statements?.get(1) as? MemberCallExpression - assertNotNull(line2) + val line1 = (bar.body as? Block)?.statements?.get(0) + assertIs(line1) + val line2 = (bar.body as? Block)?.statements?.get(1) + assertIs(line2) assertEquals(1, line1.declarations.size) val fooDecl = line1.declarations[0] @@ -554,20 +556,20 @@ class PythonFrontendTest : BaseTest() { assertNotNull(countParam) assertLocalName("c", countParam) - val countStmt = (methCount.body as? Block)?.statements?.get(0) as? IfStatement - assertNotNull(countStmt) + val countStmt = (methCount.body as? Block)?.statements?.get(0) + assertIs(countStmt) - val ifCond = countStmt.condition as? BinaryOperator - assertNotNull(ifCond) + val ifCond = countStmt.condition + assertIs(ifCond) - val lhs = ifCond.lhs as? MemberCallExpression - assertNotNull(lhs) + val lhs = ifCond.lhs + assertIs(lhs) assertEquals(countParam, (lhs.base as? Reference)?.refersTo) assertLocalName("inc", lhs) assertEquals(0, lhs.arguments.size) - val ifThen = (countStmt.thenStatement as? Block)?.statements?.get(0) as? CallExpression - assertNotNull(ifThen) + val ifThen = (countStmt.thenStatement as? Block)?.statements?.get(0) + assertIs(ifThen) assertEquals(methCount, ifThen.invokes.firstOrNull()) assertEquals(countParam, (ifThen.arguments.firstOrNull() as? Reference)?.refersTo) assertNull(countStmt.elseStatement) @@ -596,11 +598,11 @@ class PythonFrontendTest : BaseTest() { assertLocalName("self", selfReceiver) assertEquals(0, meth.parameters.size) // self is receiver and not a parameter - val methBody = meth.body as? Block - assertNotNull(methBody) + val methBody = meth.body + assertIs(methBody) - val assign = methBody.statements[0] as? AssignExpression - assertNotNull(assign) + val assign = methBody.statements[0] + assertIs(assign) val assignLhs = assign.lhs() val assignRhs = assign.rhs() @@ -616,8 +618,8 @@ class PythonFrontendTest : BaseTest() { assertNotNull(assignRhsLhs) assertEquals(selfReceiver, (assignRhsLhs.base as? Reference)?.refersTo) - val r = methBody.statements[1] as? ReturnStatement - assertNotNull(r) + val r = methBody.statements[1] + assertIs(r) assertEquals( selfReceiver, ((r.returnValue as? MemberExpression)?.base as? Reference)?.refersTo @@ -659,8 +661,8 @@ class PythonFrontendTest : BaseTest() { assertNotNull(classFieldWithInit) // classFieldNoInitializer = classFieldWithInit - val assignClsFieldOutsideFunc = clsFoo.statements[2] as? AssignExpression - assertNotNull(assignClsFieldOutsideFunc) + val assignClsFieldOutsideFunc = clsFoo.statements[2] + assertIs(assignClsFieldOutsideFunc) assertEquals( classFieldNoInitializer, (assignClsFieldOutsideFunc.lhs())?.refersTo @@ -668,43 +670,44 @@ class PythonFrontendTest : BaseTest() { assertEquals(classFieldWithInit, (assignClsFieldOutsideFunc.rhs())?.refersTo) assertEquals("=", assignClsFieldOutsideFunc.operatorCode) - val barBody = methBar.body as? Block - assertNotNull(barBody) + val barBody = methBar.body + assertIs(barBody) // self.classFieldDeclaredInFunction = 456 - val barStmt0 = barBody.statements[0] as? AssignExpression - val decl0 = barStmt0?.declarations?.get(0) as? FieldDeclaration - assertNotNull(decl0) + val barStmt0 = barBody.statements[0] + assertIs(barStmt0) + val decl0 = barStmt0.declarations[0] + assertIs(decl0) assertLocalName("classFieldDeclaredInFunction", decl0) assertNotNull(decl0.firstAssignment) // self.classFieldNoInitializer = 789 - val barStmt1 = barBody.statements[1] as? AssignExpression - assertNotNull(barStmt1) + val barStmt1 = barBody.statements[1] + assertIs(barStmt1) assertEquals(classFieldNoInitializer, (barStmt1.lhs())?.refersTo) // self.classFieldWithInit = 12 - val barStmt2 = barBody.statements[2] as? AssignExpression - assertNotNull(barStmt2) + val barStmt2 = barBody.statements[2] + assertIs(barStmt2) assertEquals(classFieldWithInit, (barStmt2.lhs())?.refersTo) // classFieldNoInitializer = "shadowed" - val barStmt3 = barBody.statements[3] as? AssignExpression - assertNotNull(barStmt3) + val barStmt3 = barBody.statements[3] + assertIs(barStmt3) assertEquals("=", barStmt3.operatorCode) assertEquals(classFieldNoInitializer, (barStmt3.lhs())?.refersTo) assertEquals("shadowed", (barStmt3.rhs>())?.value) // classFieldWithInit = "shadowed" - val barStmt4 = barBody.statements[4] as? AssignExpression - assertNotNull(barStmt4) + val barStmt4 = barBody.statements[4] + assertIs(barStmt4) assertEquals("=", barStmt4.operatorCode) assertEquals(classFieldWithInit, (barStmt4.lhs())?.refersTo) assertEquals("shadowed", (barStmt4.rhs>())?.value) // classFieldDeclaredInFunction = "shadowed" - val barStmt5 = barBody.statements[5] as? AssignExpression - assertNotNull(barStmt5) + val barStmt5 = barBody.statements[5] + assertIs(barStmt5) assertEquals("=", barStmt5.operatorCode) assertEquals(classFieldDeclaredInFunction, (barStmt5.lhs())?.refersTo) assertEquals("shadowed", (barStmt5.rhs>())?.value) @@ -757,19 +760,19 @@ class PythonFrontendTest : BaseTest() { val foo = p.variables["foo"] assertNotNull(foo) - val firstAssignment = foo.firstAssignment as? MemberCallExpression - assertNotNull(firstAssignment) + val firstAssignment = foo.firstAssignment + assertIs(firstAssignment) assertLocalName("zzz", firstAssignment) - val base = firstAssignment.base as? MemberExpression - assertNotNull(base) + val base = firstAssignment.base + assertIs(base) assertLocalName("baz", base) - val baseBase = base.base as? Reference - assertNotNull(baseBase) + val baseBase = base.base + assertIs(baseBase) assertLocalName("bar", baseBase) - val memberExpression = firstAssignment.callee as? MemberExpression - assertNotNull(memberExpression) + val memberExpression = firstAssignment.callee + assertIs(memberExpression) assertLocalName("zzz", memberExpression) } @@ -788,24 +791,24 @@ class PythonFrontendTest : BaseTest() { val main = p.functions["main"] assertNotNull(main) - val mainBody = main.body as? Block - assertNotNull(mainBody) + val mainBody = main.body + assertIs(mainBody) - val whlStmt = mainBody.statements[3] as? WhileStatement - assertNotNull(whlStmt) + val whlStmt = mainBody.statements[3] + assertIs(whlStmt) - val whlBody = whlStmt.statement as? Block - assertNotNull(whlBody) + val whlBody = whlStmt.statement + assertIs(whlBody) - val xDeclaration = whlBody.statements[0] as? AssignExpression - assertNotNull(xDeclaration) + val xDeclaration = whlBody.statements[0] + assertIs(xDeclaration) - val ifStatement = whlBody.statements[1] as? IfStatement - assertNotNull(ifStatement) + val ifStatement = whlBody.statements[1] + assertIs(ifStatement) - val brk = ifStatement.elseStatement as? Block - assertNotNull(brk) - brk.statements[0] as? BreakStatement + val brk = ifStatement.elseStatement + assertIs(brk) + assertIs(brk.statements[0]) } @Test @@ -922,14 +925,14 @@ class PythonFrontendTest : BaseTest() { val p = tu.namespaces["issue473"] assertNotNull(p) - val ifStatement = p.statements[0] as? IfStatement - assertNotNull(ifStatement) - val ifCond = ifStatement.condition as? BinaryOperator - assertNotNull(ifCond) - val ifThen = ifStatement.thenStatement as? Block - assertNotNull(ifThen) - val ifElse = ifStatement.elseStatement as? Block - assertNotNull(ifElse) + val ifStatement = p.statements[0] + assertIs(ifStatement) + val ifCond = ifStatement.condition + assertIs(ifCond) + val ifThen = ifStatement.thenStatement + assertIs(ifThen) + val ifElse = ifStatement.elseStatement + assertIs(ifElse) // sys.version_info.minor > 9 assertEquals(">", ifCond.operatorCode) @@ -940,8 +943,8 @@ class PythonFrontendTest : BaseTest() { assertNotNull(phrDeclaration) assertLocalName("phr", phrDeclaration) - val phrInitializer = phrDeclaration.firstAssignment as? BinaryOperator - assertNotNull(phrInitializer) + val phrInitializer = phrDeclaration.firstAssignment + assertIs(phrInitializer) assertEquals("|", phrInitializer.operatorCode) assertEquals(true, phrInitializer.lhs is InitializerListExpression) @@ -1038,20 +1041,20 @@ class PythonFrontendTest : BaseTest() { val varDefinedInLoop = forloopFunc.variables["varDefinedInLoop"] assertNotNull(varDefinedInLoop) - val functionBody = forloopFunc.body as? Block - assertNotNull(functionBody) + val functionBody = forloopFunc.body + assertIs(functionBody) - val firstLoop = functionBody.statements[1] as? ForEachStatement - assertNotNull(firstLoop) + val firstLoop = functionBody.statements[1] + assertIs(firstLoop) - val secondLoop = functionBody.statements[2] as? ForEachStatement - assertNotNull(secondLoop) + val secondLoop = functionBody.statements[2] + assertIs(secondLoop) - val fooCall = functionBody.statements[3] as? CallExpression - assertNotNull(fooCall) + val fooCall = functionBody.statements[3] + assertIs(fooCall) - val barCall = functionBody.statements[4] as? CallExpression - assertNotNull(barCall) + val barCall = functionBody.statements[4] + assertIs(barCall) val varDefinedBeforeLoopRef = (functionBody.statements.firstOrNull() as? AssignExpression)?.lhs?.firstOrNull() @@ -1060,8 +1063,8 @@ class PythonFrontendTest : BaseTest() { assert((firstLoop.variable?.prevDFG?.contains(varDefinedBeforeLoopRef) == false)) // dataflow from range call to loop variable - val firstLoopIterable = firstLoop.iterable as? CallExpression - assertNotNull(firstLoopIterable) + val firstLoopIterable = firstLoop.iterable + assertIs(firstLoopIterable) assert((firstLoop.variable?.prevDFG?.contains((firstLoopIterable)) == true)) // dataflow from var declaration to loop iterable call @@ -1071,16 +1074,16 @@ class PythonFrontendTest : BaseTest() { ) // dataflow from first loop to foo call - val loopVar = firstLoop.variable as? Reference - assertNotNull(loopVar) + val loopVar = firstLoop.variable + assertIs(loopVar) assertTrue(fooCall.arguments.firstOrNull()?.prevDFG?.contains(loopVar) == true) // dataflow from var declaration to foo call (in case for loop is not executed) assert(fooCall.arguments.firstOrNull()?.prevDFG?.contains(varDefinedBeforeLoopRef) == true) // dataflow from range call to loop variable - val secondLoopIterable = secondLoop.iterable as? CallExpression - assertNotNull(secondLoopIterable) + val secondLoopIterable = secondLoop.iterable + assertIs(secondLoopIterable) assert( ((secondLoop.variable as? Reference)?.prevDFG?.contains((secondLoopIterable)) == true) ) @@ -1122,8 +1125,8 @@ class PythonFrontendTest : BaseTest() { tu.allChildren() .singleOrNull { (it.lhs.singleOrNull() as? Reference)?.name?.localName == "c" } ?.rhs - ?.singleOrNull() as? BinaryOperator - assertNotNull(cAssign) + ?.singleOrNull() + assertIs(cAssign) assertEquals("or", cAssign.operatorCode) assertEquals(true, (cAssign.rhs as? Literal<*>)?.value) assertEquals("and", (cAssign.lhs as? BinaryOperator)?.operatorCode) @@ -1139,8 +1142,8 @@ class PythonFrontendTest : BaseTest() { tu.allChildren() .singleOrNull { (it.lhs.singleOrNull() as? Reference)?.name?.localName == "d" } ?.rhs - ?.singleOrNull() as? BinaryOperator - assertNotNull(dAssign) + ?.singleOrNull() + assertIs(dAssign) assertEquals("^", dAssign.operatorCode) assertEquals(0xffffL, (dAssign.rhs as? Literal<*>)?.value) assertEquals("|", (dAssign.lhs as? BinaryOperator)?.operatorCode) @@ -1160,8 +1163,8 @@ class PythonFrontendTest : BaseTest() { ?.input as? Literal<*>) ?.value ) - val dAssignLhsOfOr = (dAssign.lhs as? BinaryOperator)?.lhs as? BinaryOperator - assertNotNull(dAssignLhsOfOr) + val dAssignLhsOfOr = (dAssign.lhs as? BinaryOperator)?.lhs + assertIs(dAssignLhsOfOr) assertEquals("&", dAssignLhsOfOr.operatorCode) assertEquals("~", (dAssignLhsOfOr.rhs as? UnaryOperator)?.operatorCode) assertEquals(7L, ((dAssignLhsOfOr.rhs as? UnaryOperator)?.input as? Literal<*>)?.value) @@ -1192,34 +1195,33 @@ class PythonFrontendTest : BaseTest() { assertNotNull(tu) val namespace = tu.namespaces.singleOrNull() assertNotNull(namespace) - val aStmt = namespace.statements[0] as? AssignExpression - assertNotNull(aStmt) + val aStmt = namespace.statements[0] + assertIs(aStmt) assertEquals( "list", (aStmt.rhs.singleOrNull() as? InitializerListExpression)?.type?.name?.localName ) - val bStmt = namespace.statements[1] as? AssignExpression - assertNotNull(bStmt) + val bStmt = namespace.statements[1] + assertIs(bStmt) assertEquals( "set", (bStmt.rhs.singleOrNull() as? InitializerListExpression)?.type?.name?.localName ) - val cStmt = namespace.statements[2] as? AssignExpression - assertNotNull(cStmt) + val cStmt = namespace.statements[2] + assertIs(cStmt) assertEquals( "tuple", (cStmt.rhs.singleOrNull() as? InitializerListExpression)?.type?.name?.localName ) - val dStmt = namespace.statements[3] as? AssignExpression - assertNotNull(dStmt) + val dStmt = namespace.statements[3] + assertIs(dStmt) assertEquals( "dict", (dStmt.rhs.singleOrNull() as? InitializerListExpression)?.type?.name?.localName ) - val eStmtRhs = - (namespace.statements[4] as? AssignExpression)?.rhs?.singleOrNull() as? BinaryOperator - assertNotNull(eStmtRhs) + val eStmtRhs = (namespace.statements[4] as? AssignExpression)?.rhs?.singleOrNull() + assertIs(eStmtRhs) assertEquals("Values of a: ", (eStmtRhs.lhs as? Literal<*>)?.value) val eStmtRhsRhs = (eStmtRhs.rhs as? BinaryOperator) assertNotNull(eStmtRhsRhs) @@ -1231,10 +1233,9 @@ class PythonFrontendTest : BaseTest() { assertEquals("str", bCall?.name?.localName) assertEquals("b", bCall?.arguments?.singleOrNull()?.name?.localName) - val fStmtRhs = - (namespace.statements[5] as? AssignExpression)?.rhs?.singleOrNull() - as? SubscriptExpression - assertNotNull(fStmtRhs) + val fStmtRhs = (namespace.statements[5] as? AssignExpression)?.rhs?.singleOrNull() + + assertIs(fStmtRhs) assertEquals("a", fStmtRhs.arrayExpression.name.localName) assertTrue(fStmtRhs.subscriptExpression is RangeExpression) assertEquals( @@ -1366,19 +1367,19 @@ class PythonFrontendTest : BaseTest() { val namedExpression = result.functions["named_expression"] assertNotNull(namedExpression) - val assignExpression = result.statements[1] as? AssignExpression - assertNotNull(assignExpression) + val assignExpression = result.statements[1] + assertIs(assignExpression) assertEquals(":=", assignExpression.operatorCode) assertEquals(true, assignExpression.usedAsExpression) - val lhs = assignExpression.lhs.firstOrNull() as? Reference - assertNotNull(lhs) + val lhs = assignExpression.lhs.firstOrNull() + assertIs(lhs) val lhsVariable = lhs.refersTo as? VariableDeclaration assertLocalName("x", lhsVariable) - val rhs = assignExpression.rhs.firstOrNull() as? Literal<*> - assertNotNull(rhs) + val rhs = assignExpression.rhs.firstOrNull() + assertIs>(rhs) assertEquals(4.toLong(), rhs.evaluate()) } diff --git a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/statementHandler/StatementHandlerTest.kt b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/statementHandler/StatementHandlerTest.kt index 5c9941f544..a9927381f7 100644 --- a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/statementHandler/StatementHandlerTest.kt +++ b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/statementHandler/StatementHandlerTest.kt @@ -174,16 +174,15 @@ class StatementHandlerTest : BaseTest() { val func = result.functions["test_assert"] assertNotNull(func, "Function 'test_assert' should be found") - val assertStatement = - func.body.statements.firstOrNull { it is AssertStatement } as? AssertStatement - assertNotNull(assertStatement, "Assert statement should be found") + val assertStatement = func.body.statements.firstOrNull { it is AssertStatement } + assertIs(assertStatement, "Assert statement should be found") val condition = assertStatement.condition assertNotNull(condition, "Assert statement should have a condition") assertEquals("1 == 1", condition.code, "The condition is incorrect") - val message = assertStatement.message as? Literal<*> - assertNotNull(message, "Assert statement should have a message") + val message = assertStatement.message + assertIs>(message, "Assert statement should have a message") assertEquals("Test message", message.value, "The assert message is incorrect") } From 7ac679f4696a83ebed0a8d73a7b114328ed98541 Mon Sep 17 00:00:00 2001 From: Maximilian Kaul Date: Fri, 27 Sep 2024 12:37:15 +0200 Subject: [PATCH 25/30] refersTo -> assertRefersTo --- .../frontends/python/PythonFrontendTest.kt | 43 ++++++++----------- 1 file changed, 19 insertions(+), 24 deletions(-) diff --git a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt index ab3e0c736f..f2ec1fa2b1 100644 --- a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt +++ b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt @@ -149,7 +149,7 @@ class PythonFrontendTest : BaseTest() { assertIs(ref) assertLocalName("s", ref) - assertEquals(s, ref.refersTo) + assertRefersTo(ref, s) val stmt = compStmt.statements[1] assertIs(stmt) @@ -263,7 +263,7 @@ class PythonFrontendTest : BaseTest() { assertEquals(ctor.constructor, cls.constructors.firstOrNull()) assertFullName("simple_class.SomeClass", c1.type) - assertEquals(c1, (s2.base as? Reference)?.refersTo) + assertRefersTo((s2.base as? Reference), c1) assertEquals(1, s2.invokes.size) assertEquals(clsfunc, s2.invokes.firstOrNull()) @@ -364,7 +364,7 @@ class PythonFrontendTest : BaseTest() { val barZ = (methBar.body as? Block)?.statements?.get(0) assertIs(barZ) - assertEquals(fieldZ, barZ.refersTo) + assertRefersTo(barZ, fieldZ) val barBaz = (methBar.body as? Block)?.statements?.get(1) assertIs(barBaz) @@ -426,7 +426,7 @@ class PythonFrontendTest : BaseTest() { ?.firstOrNull() assertIs(someVarDeclaration) assertLocalName("somevar", someVarDeclaration) - assertEquals(i, (someVarDeclaration.firstAssignment as? Reference)?.refersTo) + assertRefersTo((someVarDeclaration.firstAssignment as? Reference), i) val fooMemCall = (foo.body as? Block)?.statements?.get(0) assertIs(fooMemCall) @@ -518,7 +518,7 @@ class PythonFrontendTest : BaseTest() { val initializer = fooDecl.firstAssignment as? ConstructExpression assertEquals(fooCtor, initializer?.constructor) - assertEquals(fooDecl, (line2.base as? Reference)?.refersTo) + assertRefersTo((line2.base as? Reference), fooDecl) assertEquals(foobar, line2.invokes[0]) } @@ -564,14 +564,14 @@ class PythonFrontendTest : BaseTest() { val lhs = ifCond.lhs assertIs(lhs) - assertEquals(countParam, (lhs.base as? Reference)?.refersTo) + assertRefersTo((lhs.base as? Reference), countParam) assertLocalName("inc", lhs) assertEquals(0, lhs.arguments.size) val ifThen = (countStmt.thenStatement as? Block)?.statements?.get(0) assertIs(ifThen) assertEquals(methCount, ifThen.invokes.firstOrNull()) - assertEquals(countParam, (ifThen.arguments.firstOrNull() as? Reference)?.refersTo) + assertRefersTo((ifThen.arguments.firstOrNull() as? Reference), countParam) assertNull(countStmt.elseStatement) // class c1(counter) @@ -609,21 +609,18 @@ class PythonFrontendTest : BaseTest() { assertEquals("=", assign.operatorCode) assertNotNull(assignLhs) assertNotNull(assignRhs) - assertEquals(selfReceiver, (assignLhs.base as? Reference)?.refersTo) + assertRefersTo((assignLhs.base as? Reference), selfReceiver) assertEquals("+", assignRhs.operatorCode) val assignRhsLhs = assignRhs.lhs as? MemberExpression // the second "self.total" in "self.total = self.total + 1" assertNotNull(assignRhsLhs) - assertEquals(selfReceiver, (assignRhsLhs.base as? Reference)?.refersTo) + assertRefersTo((assignRhsLhs.base as? Reference), selfReceiver) val r = methBody.statements[1] assertIs(r) - assertEquals( - selfReceiver, - ((r.returnValue as? MemberExpression)?.base as? Reference)?.refersTo - ) + assertRefersTo((r.returnValue as? MemberExpression)?.base as? Reference, selfReceiver) // TODO last line "count(c1())" } @@ -663,11 +660,8 @@ class PythonFrontendTest : BaseTest() { // classFieldNoInitializer = classFieldWithInit val assignClsFieldOutsideFunc = clsFoo.statements[2] assertIs(assignClsFieldOutsideFunc) - assertEquals( - classFieldNoInitializer, - (assignClsFieldOutsideFunc.lhs())?.refersTo - ) - assertEquals(classFieldWithInit, (assignClsFieldOutsideFunc.rhs())?.refersTo) + assertRefersTo(assignClsFieldOutsideFunc.lhs(), classFieldNoInitializer) + assertRefersTo((assignClsFieldOutsideFunc.rhs()), classFieldWithInit) assertEquals("=", assignClsFieldOutsideFunc.operatorCode) val barBody = methBar.body @@ -684,32 +678,32 @@ class PythonFrontendTest : BaseTest() { // self.classFieldNoInitializer = 789 val barStmt1 = barBody.statements[1] assertIs(barStmt1) - assertEquals(classFieldNoInitializer, (barStmt1.lhs())?.refersTo) + assertRefersTo((barStmt1.lhs()), classFieldNoInitializer) // self.classFieldWithInit = 12 val barStmt2 = barBody.statements[2] assertIs(barStmt2) - assertEquals(classFieldWithInit, (barStmt2.lhs())?.refersTo) + assertRefersTo((barStmt2.lhs()), classFieldWithInit) // classFieldNoInitializer = "shadowed" val barStmt3 = barBody.statements[3] assertIs(barStmt3) assertEquals("=", barStmt3.operatorCode) - assertEquals(classFieldNoInitializer, (barStmt3.lhs())?.refersTo) + assertRefersTo((barStmt3.lhs()), classFieldNoInitializer) assertEquals("shadowed", (barStmt3.rhs>())?.value) // classFieldWithInit = "shadowed" val barStmt4 = barBody.statements[4] assertIs(barStmt4) assertEquals("=", barStmt4.operatorCode) - assertEquals(classFieldWithInit, (barStmt4.lhs())?.refersTo) + assertRefersTo((barStmt4.lhs()), classFieldWithInit) assertEquals("shadowed", (barStmt4.rhs>())?.value) // classFieldDeclaredInFunction = "shadowed" val barStmt5 = barBody.statements[5] assertIs(barStmt5) assertEquals("=", barStmt5.operatorCode) - assertEquals(classFieldDeclaredInFunction, (barStmt5.lhs())?.refersTo) + assertRefersTo((barStmt5.lhs()), classFieldDeclaredInFunction) assertEquals("shadowed", (barStmt5.rhs>())?.value) /* TODO: @@ -1375,7 +1369,8 @@ class PythonFrontendTest : BaseTest() { val lhs = assignExpression.lhs.firstOrNull() assertIs(lhs) - val lhsVariable = lhs.refersTo as? VariableDeclaration + val lhsVariable = lhs.refersTo + assertIs(lhsVariable) assertLocalName("x", lhsVariable) val rhs = assignExpression.rhs.firstOrNull() From 6b8d8070cb676d23863414d29d8b041230ed38b0 Mon Sep 17 00:00:00 2001 From: Maximilian Kaul Date: Fri, 27 Sep 2024 12:53:00 +0200 Subject: [PATCH 26/30] code quality --- .../frontends/python/PythonFrontendTest.kt | 113 ++++++++++-------- 1 file changed, 65 insertions(+), 48 deletions(-) diff --git a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt index f2ec1fa2b1..68e88e8e16 100644 --- a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt +++ b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt @@ -57,7 +57,9 @@ class PythonFrontendTest : BaseTest() { assertNotNull(b) assertLocalName("b", b) assertEquals(assertResolvedType("bool"), b.type) - assertEquals(true, (b.firstAssignment as? Literal<*>)?.value) + val bFirstAssignment = b.firstAssignment + assertIs>(bFirstAssignment) + assertEquals(true, bFirstAssignment.value) val i = p.variables["i"] assertNotNull(i) @@ -504,10 +506,14 @@ class PythonFrontendTest : BaseTest() { assertNotNull(bar) assertLocalName("bar", bar) - assertEquals(2, (bar.body as? Block)?.statements?.size) - val line1 = (bar.body as? Block)?.statements?.get(0) + val barBody = bar.body + assertIs(barBody) + + assertEquals(2, barBody.statements.size) + val line1 = barBody.statements.get(0) assertIs(line1) - val line2 = (bar.body as? Block)?.statements?.get(1) + + val line2 = barBody.statements.get(1) assertIs(line2) assertEquals(1, line1.declarations.size) @@ -515,10 +521,11 @@ class PythonFrontendTest : BaseTest() { assertNotNull(fooDecl) assertLocalName("foo", fooDecl) assertFullName("class_ctor.Foo", fooDecl.type) - val initializer = fooDecl.firstAssignment as? ConstructExpression - assertEquals(fooCtor, initializer?.constructor) + val initializer = fooDecl.firstAssignment + assertIs(initializer) + assertEquals(fooCtor, initializer.constructor) - assertRefersTo((line2.base as? Reference), fooDecl) + assertRefersTo(line2.base, fooDecl) assertEquals(foobar, line2.invokes[0]) } @@ -556,7 +563,10 @@ class PythonFrontendTest : BaseTest() { assertNotNull(countParam) assertLocalName("c", countParam) - val countStmt = (methCount.body as? Block)?.statements?.get(0) + val methCountBody = methCount.body + assertIs(methCountBody) + + val countStmt = methCountBody.statements[0] assertIs(countStmt) val ifCond = countStmt.condition @@ -564,14 +574,16 @@ class PythonFrontendTest : BaseTest() { val lhs = ifCond.lhs assertIs(lhs) - assertRefersTo((lhs.base as? Reference), countParam) + assertRefersTo(lhs.base, countParam) assertLocalName("inc", lhs) assertEquals(0, lhs.arguments.size) - val ifThen = (countStmt.thenStatement as? Block)?.statements?.get(0) - assertIs(ifThen) - assertEquals(methCount, ifThen.invokes.firstOrNull()) - assertRefersTo((ifThen.arguments.firstOrNull() as? Reference), countParam) + val ifThenBody = countStmt.thenStatement + assertIs(ifThenBody) + val ifThenFirstStmt = ifThenBody.statements[0] + assertIs(ifThenFirstStmt) + assertEquals(methCount, ifThenFirstStmt.invokes.firstOrNull()) + assertRefersTo(ifThenFirstStmt.arguments.firstOrNull(), countParam) assertNull(countStmt.elseStatement) // class c1(counter) @@ -612,15 +624,15 @@ class PythonFrontendTest : BaseTest() { assertRefersTo((assignLhs.base as? Reference), selfReceiver) assertEquals("+", assignRhs.operatorCode) - val assignRhsLhs = - assignRhs.lhs - as? MemberExpression // the second "self.total" in "self.total = self.total + 1" - assertNotNull(assignRhsLhs) - assertRefersTo((assignRhsLhs.base as? Reference), selfReceiver) + val assignRhsLhs = assignRhs.lhs // the second "self.total" in "self.total = self.total + 1" + assertIs(assignRhsLhs) + assertRefersTo(assignRhsLhs.base, selfReceiver) val r = methBody.statements[1] assertIs(r) - assertRefersTo((r.returnValue as? MemberExpression)?.base as? Reference, selfReceiver) + val retVal = r.returnValue + assertIs(retVal) + assertRefersTo(retVal.base, selfReceiver) // TODO last line "count(c1())" } @@ -930,10 +942,13 @@ class PythonFrontendTest : BaseTest() { // sys.version_info.minor > 9 assertEquals(">", ifCond.operatorCode) - assertLocalName("minor", ifCond.lhs as? Reference) + assertIs(ifCond.lhs) + assertLocalName("minor", ifCond.lhs) // phr = {"user_id": user_id} | content - val phrDeclaration = (ifThen.statements[0] as? AssignExpression)?.declarations?.get(0) + val ifThenFirstStmt = ifThen.statements.firstOrNull() + assertIs(ifThenFirstStmt) + val phrDeclaration = ifThenFirstStmt.declarations[0] assertNotNull(phrDeclaration) assertLocalName("phr", phrDeclaration) @@ -943,7 +958,9 @@ class PythonFrontendTest : BaseTest() { assertEquals(true, phrInitializer.lhs is InitializerListExpression) // z = {"user_id": user_id} - val elseStmt1 = (ifElse.statements[0] as? AssignExpression)?.declarations?.get(0) + val elseFirstStmt = ifElse.statements.firstOrNull() + assertIs(elseFirstStmt) + val elseStmt1 = elseFirstStmt.declarations[0] assertNotNull(elseStmt1) assertLocalName("z", elseStmt1) @@ -1050,9 +1067,11 @@ class PythonFrontendTest : BaseTest() { val barCall = functionBody.statements[4] assertIs(barCall) - val varDefinedBeforeLoopRef = - (functionBody.statements.firstOrNull() as? AssignExpression)?.lhs?.firstOrNull() - as? Reference ?: TODO() + val bodyFirstStmt = functionBody.statements.firstOrNull() + assertIs(bodyFirstStmt) + val varDefinedBeforeLoopRef = bodyFirstStmt.lhs.firstOrNull() + assertIs(varDefinedBeforeLoopRef) + // no dataflow from var declaration to loop variable because it's a write access assert((firstLoop.variable?.prevDFG?.contains(varDefinedBeforeLoopRef) == false)) @@ -1078,15 +1097,13 @@ class PythonFrontendTest : BaseTest() { // dataflow from range call to loop variable val secondLoopIterable = secondLoop.iterable assertIs(secondLoopIterable) - assert( - ((secondLoop.variable as? Reference)?.prevDFG?.contains((secondLoopIterable)) == true) - ) + + val secondLoopVar = secondLoop.variable + assertIs(secondLoopVar) + assert(secondLoopVar.prevDFG.contains(secondLoopIterable) == true) // dataflow from second loop var to bar call - assertEquals( - (secondLoop.variable as? Reference), - barCall.arguments.firstOrNull()?.prevDFG?.firstOrNull() - ) + assertEquals(secondLoopVar, barCall.arguments.firstOrNull()?.prevDFG?.firstOrNull()) } @Test @@ -1189,30 +1206,30 @@ class PythonFrontendTest : BaseTest() { assertNotNull(tu) val namespace = tu.namespaces.singleOrNull() assertNotNull(namespace) + val aStmt = namespace.statements[0] assertIs(aStmt) - assertEquals( - "list", - (aStmt.rhs.singleOrNull() as? InitializerListExpression)?.type?.name?.localName - ) + val aStmtRhs = aStmt.rhs.singleOrNull() + assertIs(aStmtRhs) + assertEquals("list", aStmtRhs.type.name.localName) + val bStmt = namespace.statements[1] assertIs(bStmt) - assertEquals( - "set", - (bStmt.rhs.singleOrNull() as? InitializerListExpression)?.type?.name?.localName - ) + val bStmtRhs = bStmt.rhs.singleOrNull() + assertIs(bStmtRhs) + assertEquals("set", bStmtRhs.type.name.localName) + val cStmt = namespace.statements[2] assertIs(cStmt) - assertEquals( - "tuple", - (cStmt.rhs.singleOrNull() as? InitializerListExpression)?.type?.name?.localName - ) + val cStmtRhs = cStmt.rhs.singleOrNull() + assertIs(cStmtRhs) + assertEquals("tuple", cStmtRhs.type.name.localName) + val dStmt = namespace.statements[3] assertIs(dStmt) - assertEquals( - "dict", - (dStmt.rhs.singleOrNull() as? InitializerListExpression)?.type?.name?.localName - ) + val dStmtRhs = dStmt.rhs.singleOrNull() + assertIs(dStmtRhs) + assertEquals("dict", dStmtRhs.type.name.localName) val eStmtRhs = (namespace.statements[4] as? AssignExpression)?.rhs?.singleOrNull() assertIs(eStmtRhs) From 12777e853129b4db23dc850b15374b3985080585 Mon Sep 17 00:00:00 2001 From: Maximilian Kaul Date: Fri, 27 Sep 2024 12:56:30 +0200 Subject: [PATCH 27/30] assertInvokes --- .../aisec/cpg/frontends/python/PythonFrontendTest.kt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt index 68e88e8e16..eb62e1aefa 100644 --- a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt +++ b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt @@ -119,7 +119,7 @@ class PythonFrontendTest : BaseTest() { assertIs(callExpression) assertLocalName("bar", callExpression) - assertEquals(bar, callExpression.invokes.firstOrNull()) + assertInvokes(callExpression, bar) val edge = callExpression.argumentEdges[1] assertNotNull(edge) @@ -267,7 +267,7 @@ class PythonFrontendTest : BaseTest() { assertRefersTo((s2.base as? Reference), c1) assertEquals(1, s2.invokes.size) - assertEquals(clsfunc, s2.invokes.firstOrNull()) + assertInvokes(s2, clsfunc) // member } @@ -439,7 +439,7 @@ class PythonFrontendTest : BaseTest() { assertEquals(".", fooMemCall.operatorCode) assertFullName("class_self.Foo.bar", fooMemCall) assertEquals(1, fooMemCall.invokes.size) - assertEquals(bar, fooMemCall.invokes[0]) + assertInvokes(fooMemCall, bar) assertLocalName("self", fooMemCall.base) } @@ -526,7 +526,7 @@ class PythonFrontendTest : BaseTest() { assertEquals(fooCtor, initializer.constructor) assertRefersTo(line2.base, fooDecl) - assertEquals(foobar, line2.invokes[0]) + assertInvokes(line2, foobar) } @Test @@ -582,7 +582,7 @@ class PythonFrontendTest : BaseTest() { assertIs(ifThenBody) val ifThenFirstStmt = ifThenBody.statements[0] assertIs(ifThenFirstStmt) - assertEquals(methCount, ifThenFirstStmt.invokes.firstOrNull()) + assertInvokes(ifThenFirstStmt, methCount) assertRefersTo(ifThenFirstStmt.arguments.firstOrNull(), countParam) assertNull(countStmt.elseStatement) From 0640490122c4211adcae80fcd4033d2eaa49d466 Mon Sep 17 00:00:00 2001 From: Maximilian Kaul Date: Fri, 27 Sep 2024 12:58:17 +0200 Subject: [PATCH 28/30] assertInvokes --- .../frontends/python/statementHandler/StatementHandlerTest.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/statementHandler/StatementHandlerTest.kt b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/statementHandler/StatementHandlerTest.kt index a9927381f7..8fe770c63d 100644 --- a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/statementHandler/StatementHandlerTest.kt +++ b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/statementHandler/StatementHandlerTest.kt @@ -154,7 +154,7 @@ class StatementHandlerTest : BaseTest() { val add = result.operators["__add__"] assertNotNull(add) - assertEquals(add, opCall.invokes.singleOrNull()) + assertInvokes(opCall, add) // ... and one to __pos__ (+) opCall = result.operatorCalls("+").getOrNull(1) @@ -163,7 +163,7 @@ class StatementHandlerTest : BaseTest() { val pos = result.operators["__pos__"] assertNotNull(pos) - assertEquals(pos, opCall.invokes.singleOrNull()) + assertInvokes(opCall, pos) } } From a7ff213c94178f100b77780d70aa61d04e5cce72 Mon Sep 17 00:00:00 2001 From: Maximilian Kaul Date: Fri, 27 Sep 2024 13:09:00 +0200 Subject: [PATCH 29/30] replace more as? with assertIs --- .../frontends/python/PythonFrontendTest.kt | 248 +++++++++++------- 1 file changed, 152 insertions(+), 96 deletions(-) diff --git a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt index eb62e1aefa..d4e788590b 100644 --- a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt +++ b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt @@ -65,13 +65,17 @@ class PythonFrontendTest : BaseTest() { assertNotNull(i) assertLocalName("i", i) assertEquals(assertResolvedType("int"), i.type) - assertEquals(42L, (i.firstAssignment as? Literal<*>)?.value) + val iFirstAssignment = i.firstAssignment + assertIs>(iFirstAssignment) + assertEquals(42L, iFirstAssignment.value) val f = p.variables["f"] assertNotNull(f) assertLocalName("f", f) assertEquals(assertResolvedType("float"), f.type) - assertEquals(1.0, (f.firstAssignment as? Literal<*>)?.value) + val fFirstAssignment = f.firstAssignment + assertIs>(fFirstAssignment) + assertEquals(1.0, fFirstAssignment.value) val c = p.variables["c"] assertNotNull(c) @@ -84,13 +88,17 @@ class PythonFrontendTest : BaseTest() { assertNotNull(t) assertLocalName("t", t) assertEquals(assertResolvedType("str"), t.type) - assertEquals("Hello", (t.firstAssignment as? Literal<*>)?.value) + val tAssignment = t.firstAssignment + assertIs>(tAssignment) + assertEquals("Hello", tAssignment.value) val n = p.variables["n"] assertNotNull(n) assertLocalName("n", n) assertEquals(assertResolvedType("None"), n.type) - assertEquals(null, (n.firstAssignment as? Literal<*>)?.value) + val nAssignment = n.firstAssignment + assertIs>(nAssignment) + assertEquals(null, nAssignment.value) } } @@ -169,12 +177,16 @@ class PythonFrontendTest : BaseTest() { val lhs = op.lhs assertIs>(lhs) - assertEquals(1, (lhs.value as? Long)?.toInt()) + val lhsValue = lhs.value + assertIs(lhsValue) + assertEquals(1, lhsValue.toInt()) val rhs = op.rhs assertIs>(rhs) - assertEquals(2, (rhs.value as? Long)?.toInt()) + val rhsValue = rhs.value + assertIs(rhsValue) + assertEquals(2, rhsValue.toInt()) val r = compStmt.statements[3] assertIs(r) @@ -204,7 +216,9 @@ class PythonFrontendTest : BaseTest() { val body = main.body assertIs(body) - val sel = (body.statements.firstOrNull() as? AssignExpression)?.declarations?.firstOrNull() + val bodyFirstStmt = body.statements.firstOrNull() + assertIs(bodyFirstStmt) + val sel = bodyFirstStmt.declarations.firstOrNull() assertNotNull(sel) assertLocalName("sel", sel) assertEquals(tu.primitiveType("bool"), sel.type) @@ -265,7 +279,7 @@ class PythonFrontendTest : BaseTest() { assertEquals(ctor.constructor, cls.constructors.firstOrNull()) assertFullName("simple_class.SomeClass", c1.type) - assertRefersTo((s2.base as? Reference), c1) + assertRefersTo(s2.base, c1) assertEquals(1, s2.invokes.size) assertInvokes(s2, clsfunc) @@ -285,7 +299,9 @@ class PythonFrontendTest : BaseTest() { val main = p.functions["foo"] assertNotNull(main) - val assignExpr = (main.body as? Block)?.statements?.firstOrNull() + val mainBody = main.body + assertIs(mainBody) + val assignExpr = mainBody.statements.firstOrNull() assertIs(assignExpr) val foo = assignExpr.declarations.firstOrNull() @@ -308,10 +324,14 @@ class PythonFrontendTest : BaseTest() { assertEquals(false, ifCond.value) assertEquals(tu.primitiveType("int"), thenExpr.type) - assertEquals(21, (thenExpr.value as? Long)?.toInt()) + val thenValue = thenExpr.value + assertIs(thenValue) + assertEquals(21, thenValue.toInt()) + val elseValue = elseExpr.value + assertIs(elseValue) assertEquals(tu.primitiveType("int"), elseExpr.type) - assertEquals(42, (elseExpr.value as? Long)?.toInt()) + assertEquals(42, elseValue.toInt()) } @Test @@ -364,11 +384,13 @@ class PythonFrontendTest : BaseTest() { assertNotNull(methBar) assertLocalName("bar", methBar) - val barZ = (methBar.body as? Block)?.statements?.get(0) + val methBarBody = methBar.body + assertIs(methBarBody) + val barZ = methBarBody.statements[0] assertIs(barZ) assertRefersTo(barZ, fieldZ) - val barBaz = (methBar.body as? Block)?.statements?.get(1) + val barBaz = methBarBody.statements[1] assertIs(barBaz) val barBazInner = barBaz.declarations[0] assertIs(barBazInner) @@ -422,15 +444,18 @@ class PythonFrontendTest : BaseTest() { // assertEquals(tu.primitiveType("int"), i.type) // self.somevar = i - val someVarDeclaration = - ((bar.body as? Block)?.statements?.get(0) as? AssignExpression) - ?.declarations - ?.firstOrNull() + val barBody = bar.body + assertIs(barBody) + val barBodyFirstStmt = barBody.statements[0] + assertIs(barBodyFirstStmt) + val someVarDeclaration = barBodyFirstStmt.declarations.firstOrNull() assertIs(someVarDeclaration) assertLocalName("somevar", someVarDeclaration) - assertRefersTo((someVarDeclaration.firstAssignment as? Reference), i) + assertRefersTo(someVarDeclaration.firstAssignment, i) - val fooMemCall = (foo.body as? Block)?.statements?.get(0) + val fooBody = foo.body + assertIs(fooBody) + val fooMemCall = fooBody.statements[0] assertIs(fooMemCall) val mem = fooMemCall.callee @@ -510,10 +535,10 @@ class PythonFrontendTest : BaseTest() { assertIs(barBody) assertEquals(2, barBody.statements.size) - val line1 = barBody.statements.get(0) + val line1 = barBody.statements[0] assertIs(line1) - val line2 = barBody.statements.get(1) + val line2 = barBody.statements[1] assertIs(line2) assertEquals(1, line1.declarations.size) @@ -588,10 +613,9 @@ class PythonFrontendTest : BaseTest() { // class c1(counter) assertLocalName("c1", clsC1) - assertEquals( - clsCounter, - (clsC1.superClasses.firstOrNull() as? ObjectType)?.recordDeclaration - ) + val cls1Super = clsC1.superClasses.firstOrNull() + assertIs(cls1Super) + assertEquals(clsCounter, cls1Super.recordDeclaration) assertEquals(1, clsC1.fields.size) val field = clsC1.fields[0] @@ -621,7 +645,7 @@ class PythonFrontendTest : BaseTest() { assertEquals("=", assign.operatorCode) assertNotNull(assignLhs) assertNotNull(assignRhs) - assertRefersTo((assignLhs.base as? Reference), selfReceiver) + assertRefersTo(assignLhs.base, selfReceiver) assertEquals("+", assignRhs.operatorCode) val assignRhsLhs = assignRhs.lhs // the second "self.total" in "self.total = self.total + 1" @@ -1123,73 +1147,96 @@ class PythonFrontendTest : BaseTest() { val bAugAssign = tu.allChildren().singleOrNull { - (it.lhs.singleOrNull() as? Reference)?.name?.localName == "b" && - it.location?.region?.startLine == 4 + val itLhs = it.lhs.singleOrNull() + assertIs(itLhs) + itLhs.name.localName == "b" && it.location?.region?.startLine == 4 } assertNotNull(bAugAssign) assertEquals("*=", bAugAssign.operatorCode) assertEquals("b", bAugAssign.lhs.singleOrNull()?.name?.localName) - assertEquals(2L, (bAugAssign.rhs.singleOrNull() as? Literal<*>)?.value) + val bAugAssignRhs = bAugAssign.rhs.singleOrNull() + assertIs>(bAugAssignRhs) + assertEquals(2L, bAugAssignRhs.value) // c = (not True and False) or True val cAssign = tu.allChildren() - .singleOrNull { (it.lhs.singleOrNull() as? Reference)?.name?.localName == "c" } + .singleOrNull { + val itLhs = it.lhs.singleOrNull() + assertIs(itLhs) + itLhs.name.localName == "c" + } ?.rhs ?.singleOrNull() assertIs(cAssign) assertEquals("or", cAssign.operatorCode) - assertEquals(true, (cAssign.rhs as? Literal<*>)?.value) - assertEquals("and", (cAssign.lhs as? BinaryOperator)?.operatorCode) - assertEquals(false, ((cAssign.lhs as? BinaryOperator)?.rhs as? Literal<*>)?.value) - assertEquals("not", ((cAssign.lhs as? BinaryOperator)?.lhs as? UnaryOperator)?.operatorCode) - assertEquals( - true, - (((cAssign.lhs as? BinaryOperator)?.lhs as? UnaryOperator)?.input as? Literal<*>)?.value - ) + val cAssignRhs = cAssign.rhs + assertIs>(cAssignRhs) + assertEquals(true, cAssignRhs.value) + val cAssignLhs = cAssign.lhs + assertIs(cAssignLhs) + assertEquals("and", cAssignLhs.operatorCode) + val cAssignLhsRhs = cAssignLhs.rhs + assertIs>(cAssignLhsRhs) + assertEquals(false, cAssignLhsRhs.value) + val cAssignLhsLhs = cAssignLhs.lhs + assertIs(cAssignLhsLhs) + assertEquals("not", cAssignLhsLhs.operatorCode) + val cAssignLhsLhsInput = cAssignLhsLhs.input + assertIs>(cAssignLhsLhsInput) + assertEquals(true, cAssignLhsLhsInput.value) // d = ((-5 >> 2) & ~7 | (+4 << 1)) ^ 0xffff val dAssign = tu.allChildren() - .singleOrNull { (it.lhs.singleOrNull() as? Reference)?.name?.localName == "d" } + .singleOrNull { + val itLhs = it.lhs.singleOrNull() + assertIs(itLhs) + itLhs.name.localName == "d" + } ?.rhs ?.singleOrNull() assertIs(dAssign) assertEquals("^", dAssign.operatorCode) - assertEquals(0xffffL, (dAssign.rhs as? Literal<*>)?.value) - assertEquals("|", (dAssign.lhs as? BinaryOperator)?.operatorCode) - assertEquals("<<", ((dAssign.lhs as? BinaryOperator)?.rhs as? BinaryOperator)?.operatorCode) - assertEquals( - 1L, - (((dAssign.lhs as? BinaryOperator)?.rhs as? BinaryOperator)?.rhs as? Literal<*>)?.value - ) - assertEquals( - "+", - (((dAssign.lhs as? BinaryOperator)?.rhs as? BinaryOperator)?.lhs as? UnaryOperator) - ?.operatorCode - ) - assertEquals( - 4L, - ((((dAssign.lhs as? BinaryOperator)?.rhs as? BinaryOperator)?.lhs as? UnaryOperator) - ?.input as? Literal<*>) - ?.value - ) - val dAssignLhsOfOr = (dAssign.lhs as? BinaryOperator)?.lhs + val dAssignRhs = dAssign.rhs + assertIs>(dAssignRhs) + assertEquals(0xffffL, dAssignRhs.value) + val dAssignLhs = dAssign.lhs + assertIs(dAssignLhs) + assertEquals("|", dAssignLhs.operatorCode) + val dAssignLhsRhs = dAssignLhs.rhs + assertIs(dAssignLhsRhs) + assertEquals("<<", dAssignLhsRhs.operatorCode) + val dAssignLhsRhsRhs = dAssignLhsRhs.rhs + assertIs>(dAssignLhsRhsRhs) + assertEquals(1L, dAssignLhsRhsRhs.value) + val dAssignLhsRhsLhs = dAssignLhsRhs.lhs + assertIs(dAssignLhsRhsLhs) + assertEquals("+", dAssignLhsRhsLhs.operatorCode) + val dAssignLhsRhsLhsInput = dAssignLhsRhsLhs.input + assertIs>(dAssignLhsRhsLhsInput) + assertEquals(4L, dAssignLhsRhsLhsInput.value) + val dAssignLhsOfOr = dAssignLhs.lhs assertIs(dAssignLhsOfOr) assertEquals("&", dAssignLhsOfOr.operatorCode) - assertEquals("~", (dAssignLhsOfOr.rhs as? UnaryOperator)?.operatorCode) - assertEquals(7L, ((dAssignLhsOfOr.rhs as? UnaryOperator)?.input as? Literal<*>)?.value) - assertEquals(">>", (dAssignLhsOfOr.lhs as? BinaryOperator)?.operatorCode) - assertEquals(2L, ((dAssignLhsOfOr.lhs as? BinaryOperator)?.rhs as? Literal<*>)?.value) - assertEquals( - "-", - ((dAssignLhsOfOr.lhs as? BinaryOperator)?.lhs as? UnaryOperator)?.operatorCode - ) - assertEquals( - 5L, - (((dAssignLhsOfOr.lhs as? BinaryOperator)?.lhs as? UnaryOperator)?.input as? Literal<*>) - ?.value - ) + val dAssignLhsOfOrRhs = dAssignLhsOfOr.rhs + assertIs(dAssignLhsOfOrRhs) + assertEquals("~", dAssignLhsOfOrRhs.operatorCode) + val dAssignLhsOfOrRhsInput = dAssignLhsOfOrRhs.input + assertIs>(dAssignLhsOfOrRhsInput) + assertEquals(7L, dAssignLhsOfOrRhsInput.value) + val dAssignLhsOfOrLhs = dAssignLhsOfOr.lhs + assertIs(dAssignLhsOfOrLhs) + assertEquals(">>", dAssignLhsOfOrLhs.operatorCode) + val dAssignLhsOfOrLhsRhs = dAssignLhsOfOrLhs.rhs + assertIs>(dAssignLhsOfOrLhsRhs) + assertEquals(2L, dAssignLhsOfOrLhsRhs.value) + val dAssignLhsOfOrLhsLhs = dAssignLhsOfOrLhs.lhs + assertIs(dAssignLhsOfOrLhsLhs) + assertEquals("-", dAssignLhsOfOrLhsLhs.operatorCode) + val dAssignLhsOfOrLhsLhsInput = dAssignLhsOfOrLhsLhs.input + assertIs>(dAssignLhsOfOrLhsLhsInput) + assertEquals(5L, dAssignLhsOfOrLhsLhsInput.value) } @Test @@ -1231,36 +1278,45 @@ class PythonFrontendTest : BaseTest() { assertIs(dStmtRhs) assertEquals("dict", dStmtRhs.type.name.localName) - val eStmtRhs = (namespace.statements[4] as? AssignExpression)?.rhs?.singleOrNull() + val fourthStmt = namespace.statements[4] + assertIs(fourthStmt) + val eStmtRhs = fourthStmt.rhs.singleOrNull() assertIs(eStmtRhs) - assertEquals("Values of a: ", (eStmtRhs.lhs as? Literal<*>)?.value) - val eStmtRhsRhs = (eStmtRhs.rhs as? BinaryOperator) + val eStmtRhsLhs = eStmtRhs.lhs + assertIs>(eStmtRhsLhs) + assertEquals("Values of a: ", eStmtRhsLhs.value) + val eStmtRhsRhs = eStmtRhs.rhs + assertIs(eStmtRhsRhs) assertNotNull(eStmtRhsRhs) - val aRef = eStmtRhsRhs.lhs as? Reference - assertEquals("a", aRef?.name?.localName) - val eStmtRhsRhsRhs = (eStmtRhsRhs.rhs as? BinaryOperator) - assertEquals(" and b: ", (eStmtRhsRhsRhs?.lhs as? Literal<*>)?.value) - val bCall = eStmtRhsRhsRhs?.rhs as? CallExpression - assertEquals("str", bCall?.name?.localName) - assertEquals("b", bCall?.arguments?.singleOrNull()?.name?.localName) - - val fStmtRhs = (namespace.statements[5] as? AssignExpression)?.rhs?.singleOrNull() + val aRef = eStmtRhsRhs.lhs + assertEquals("a", aRef.name.localName) + val eStmtRhsRhsRhs = eStmtRhsRhs.rhs + assertIs(eStmtRhsRhsRhs) + val eStmtRhsRhsRhsLhs = eStmtRhsRhsRhs.lhs + assertIs>(eStmtRhsRhsRhsLhs) + assertEquals(" and b: ", eStmtRhsRhsRhsLhs.value) + val bCall = eStmtRhsRhsRhs.rhs + assertIs(bCall) + assertEquals("str", bCall.name.localName) + assertEquals("b", bCall.arguments.singleOrNull()?.name?.localName) + + val fifthStmt = namespace.statements[5] + assertIs(fifthStmt) + val fStmtRhs = fifthStmt.rhs.singleOrNull() assertIs(fStmtRhs) assertEquals("a", fStmtRhs.arrayExpression.name.localName) - assertTrue(fStmtRhs.subscriptExpression is RangeExpression) - assertEquals( - 1L, - ((fStmtRhs.subscriptExpression as RangeExpression).floor as? Literal<*>)?.value - ) - assertEquals( - 3L, - ((fStmtRhs.subscriptExpression as RangeExpression).ceiling as? Literal<*>)?.value - ) - assertEquals( - 2L, - ((fStmtRhs.subscriptExpression as RangeExpression).third as? Literal<*>)?.value - ) + val subscriptExpression = fStmtRhs.subscriptExpression + assertIs(subscriptExpression) + val fStmtRhsFloor = subscriptExpression.floor + assertIs>(fStmtRhsFloor) + assertEquals(1L, fStmtRhsFloor.value) + val fStmtRhsCeiling = subscriptExpression.ceiling + assertIs>(fStmtRhsCeiling) + assertEquals(3L, fStmtRhsCeiling.value) + val fStmtRhsThird = subscriptExpression.third + assertIs>(fStmtRhsThird) + assertEquals(2L, fStmtRhsThird.value) } @Test From 2ba3f87fe6b54080c4a2f5965c45265aa16a82c1 Mon Sep 17 00:00:00 2001 From: Maximilian Kaul Date: Fri, 27 Sep 2024 13:48:56 +0200 Subject: [PATCH 30/30] is to assertIs --- .../aisec/cpg/frontends/python/PythonFrontendTest.kt | 3 ++- .../frontends/python/statementHandler/StatementHandlerTest.kt | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt index d4e788590b..4873153a00 100644 --- a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt +++ b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt @@ -979,7 +979,8 @@ class PythonFrontendTest : BaseTest() { val phrInitializer = phrDeclaration.firstAssignment assertIs(phrInitializer) assertEquals("|", phrInitializer.operatorCode) - assertEquals(true, phrInitializer.lhs is InitializerListExpression) + val phrInitializerLhs = phrInitializer.lhs + assertIs(phrInitializerLhs) // z = {"user_id": user_id} val elseFirstStmt = ifElse.statements.firstOrNull() diff --git a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/statementHandler/StatementHandlerTest.kt b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/statementHandler/StatementHandlerTest.kt index 8fe770c63d..f13e054207 100644 --- a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/statementHandler/StatementHandlerTest.kt +++ b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/statementHandler/StatementHandlerTest.kt @@ -201,13 +201,13 @@ class StatementHandlerTest : BaseTest() { // Test for `del my_list[2]` val deleteStmt2 = deleteExpressions[1] assertEquals(1, deleteStmt2.operands.size) - assertTrue(deleteStmt2.operands.firstOrNull() is SubscriptExpression) + assertIs(deleteStmt2.operands.firstOrNull()) assertTrue(deleteStmt2.additionalProblems.isEmpty()) // Test for `del my_dict['b']` val deleteStmt3 = deleteExpressions[2] assertEquals(1, deleteStmt3.operands.size) - assertTrue(deleteStmt3.operands.firstOrNull() is SubscriptExpression) + assertIs(deleteStmt3.operands.firstOrNull()) assertTrue(deleteStmt3.additionalProblems.isEmpty()) // Test for `del obj.d`