diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/TryStatement.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/TryStatement.kt index 86df0074e7..1f887fa207 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/TryStatement.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/TryStatement.kt @@ -35,16 +35,44 @@ import org.neo4j.ogm.annotation.Relationship /** A [Statement] which represents a try/catch block, primarily used for exception handling. */ class TryStatement : Statement() { + /** + * This represents some kind of resource which is typically opened (or similar) while entering + * the [tryBlock]. If this operation fails, we may continue with the [finallyBlock]. However, + * there is no exception raised but if an exception occurs while opening the resource, we won't + * enter the [tryBlock]. + */ @Relationship(value = "RESOURCES", direction = Relationship.Direction.OUTGOING) var resourceEdges = astEdgesOf() var resources by unwrapping(TryStatement::resourceEdges) + /** + * This represents a block whose statements can throw exceptions which are handled by the + * [catchClauses]. + */ @Relationship(value = "TRY_BLOCK") var tryBlockEdge = astOptionalEdgeOf() var tryBlock by unwrapping(TryStatement::tryBlockEdge) + /** + * This represents a block whose statements are only executed if the [tryBlock] finished without + * exceptions. Note that any exception thrown in this block is no longer caught by the + * [catchClauses]. + */ + @Relationship(value = "ELSE_BLOCK") var elseBlockEdge = astOptionalEdgeOf() + var elseBlock by unwrapping(TryStatement::elseBlockEdge) + + /** + * This represents a block of statements which is always executed after finishing the [tryBlock] + * or one of the [catchClauses]. Note that any exception thrown in this block is no longer + * caught by the [catchClauses]. + */ @Relationship(value = "FINALLY_BLOCK") var finallyBlockEdge = astOptionalEdgeOf() var finallyBlock by unwrapping(TryStatement::finallyBlockEdge) + /** + * This represents a set of blocks whose statements handle the exceptions which are thrown in + * the [tryBlock]. There can be multiple catch clauses, but it is also possible that none + * exists. + */ @Relationship(value = "CATCH_CLAUSES", direction = Relationship.Direction.OUTGOING) var catchClauseEdges = astEdgesOf() var catchClauses by unwrapping(TryStatement::catchClauseEdges) @@ -58,9 +86,10 @@ class TryStatement : Statement() { tryBlock == other.tryBlock && finallyBlock == other.finallyBlock && catchClauses == other.catchClauses && + elseBlock == other.elseBlock && propertyEqualsList(catchClauseEdges, other.catchClauseEdges)) } override fun hashCode() = - Objects.hash(super.hashCode(), resources, tryBlock, finallyBlock, catchClauses) + Objects.hash(super.hashCode(), resources, tryBlock, finallyBlock, catchClauses, elseBlock) } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/EvaluationOrderGraphPass.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/EvaluationOrderGraphPass.kt index ab671ba2df..5b64513ea5 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/EvaluationOrderGraphPass.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/EvaluationOrderGraphPass.kt @@ -450,7 +450,7 @@ open class EvaluationOrderGraphPass(ctx: TranslationContext) : TranslationUnitPa // existing function and the EOG handler for handling function declarations will // reset the // stack - val oldEOG = ArrayList(currentPredecessors) + val oldEOG = currentPredecessors.toMutableList() // analyze the defaults createEOG(declaration) @@ -559,9 +559,9 @@ open class EvaluationOrderGraphPass(ctx: TranslationContext) : TranslationUnitPa val throwType = input.type pushToEOG(node) if (catchingScope is TryScope) { - catchingScope.catchesOrRelays[throwType] = ArrayList(currentPredecessors) + catchingScope.catchesOrRelays[throwType] = currentPredecessors.toMutableList() } else if (catchingScope is FunctionScope) { - catchingScope.catchesOrRelays[throwType] = ArrayList(currentPredecessors) + catchingScope.catchesOrRelays[throwType] = currentPredecessors.toMutableList() } currentPredecessors.clear() } @@ -581,7 +581,7 @@ open class EvaluationOrderGraphPass(ctx: TranslationContext) : TranslationUnitPa protected fun handleAssertStatement(node: AssertStatement) { createEOG(node.condition) - val openConditionEOGs = ArrayList(currentPredecessors) + val openConditionEOGs = currentPredecessors.toMutableList() createEOG(node.message) setCurrentEOGs(openConditionEOGs) pushToEOG(node) @@ -598,7 +598,8 @@ open class EvaluationOrderGraphPass(ctx: TranslationContext) : TranslationUnitPa node.resources.forEach { createEOG(it) } createEOG(node.tryBlock) - val tmpEOGNodes = ArrayList(currentPredecessors) + val tmpEOGNodes = currentPredecessors.toMutableList() + val catchEnds = mutableListOf() val catchesOrRelays = tryScope?.catchesOrRelays for (catchClause in node.catchClauses) { currentPredecessors.clear() @@ -617,8 +618,21 @@ open class EvaluationOrderGraphPass(ctx: TranslationContext) : TranslationUnitPa toRemove.forEach { catchesOrRelays?.remove(it) } pushToEOG(catchClause) createEOG(catchClause.body) + catchEnds.addAll(currentPredecessors) + } + + // We need to handle the else block after the catch clauses, as the else could contain a + // throw itself + // that should not be caught be the catch clauses. + if (node.elseBlock != null) { + currentPredecessors.clear() + currentPredecessors.addAll(tmpEOGNodes) + createEOG(node.elseBlock) + // All valid try ends got through the else block. + tmpEOGNodes.clear() tmpEOGNodes.addAll(currentPredecessors) } + tmpEOGNodes.addAll(catchEnds) val canTerminateExceptionfree = tmpEOGNodes.any { reachableFromValidEOGRoot(it) } currentPredecessors.clear() @@ -650,7 +664,7 @@ open class EvaluationOrderGraphPass(ctx: TranslationContext) : TranslationUnitPa if (outerScope is TryScope) outerScope.catchesOrRelays else (outerScope as FunctionScope).catchesOrRelays for ((key, value) in catchesOrRelays ?: mapOf()) { - val catches = outerCatchesOrRelays[key] ?: ArrayList() + val catches = outerCatchesOrRelays[key] ?: mutableListOf() catches.addAll(value) outerCatchesOrRelays[key] = catches } @@ -780,7 +794,7 @@ open class EvaluationOrderGraphPass(ctx: TranslationContext) : TranslationUnitPa fun setCurrentEOGs(nodes: List) { LOGGER.trace("Setting {} to EOGs", nodes) - currentPredecessors = ArrayList(nodes) + currentPredecessors = nodes.toMutableList() } /** @@ -793,7 +807,7 @@ open class EvaluationOrderGraphPass(ctx: TranslationContext) : TranslationUnitPa // Breaks are connected to the NEXT EOG node and therefore temporarily stored after the loop // context is destroyed currentPredecessors.addAll(loopScope.breakStatements) - val continues = ArrayList(loopScope.continueStatements) + val continues = loopScope.continueStatements.toMutableList() if (continues.isNotEmpty()) { val conditions = loopScope.conditions.map { SubgraphWalker.getEOGPathEdges(it).entries }.flatten() @@ -845,7 +859,7 @@ open class EvaluationOrderGraphPass(ctx: TranslationContext) : TranslationUnitPa createEOG(node.condition) // To have semantic information after the condition evaluation pushToEOG(node) - val openConditionEOGs = ArrayList(currentPredecessors) + val openConditionEOGs = currentPredecessors.toMutableList() nextEdgeBranch = true createEOG(node.thenExpression) openBranchNodes.addAll(currentPredecessors) @@ -882,7 +896,7 @@ open class EvaluationOrderGraphPass(ctx: TranslationContext) : TranslationUnitPa node.variable?.let { node.prevDFGEdges += it } pushToEOG(node) // To have semantic information after the variable declaration nextEdgeBranch = true - val tmpEOGNodes = ArrayList(currentPredecessors) + val tmpEOGNodes = currentPredecessors.toMutableList() createEOG(node.statement) connectCurrentToLoopStart() currentPredecessors.clear() @@ -904,7 +918,7 @@ open class EvaluationOrderGraphPass(ctx: TranslationContext) : TranslationUnitPa pushToEOG(node) // To have semantic information after the condition evaluation nextEdgeBranch = true - val tmpEOGNodes = ArrayList(currentPredecessors) + val tmpEOGNodes = currentPredecessors.toMutableList() createEOG(node.statement) createEOG(node.iterationStatement) @@ -929,7 +943,7 @@ open class EvaluationOrderGraphPass(ctx: TranslationContext) : TranslationUnitPa createEOG(node.conditionDeclaration) createEOG(node.condition) pushToEOG(node) // To have semantic information after the condition evaluation - val openConditionEOGs = ArrayList(currentPredecessors) + val openConditionEOGs = currentPredecessors.toMutableList() nextEdgeBranch = true createEOG(node.thenStatement) openBranchNodes.addAll(currentPredecessors) @@ -951,7 +965,7 @@ open class EvaluationOrderGraphPass(ctx: TranslationContext) : TranslationUnitPa createEOG(node.selectorDeclaration) createEOG(node.selector) pushToEOG(node) // To have semantic information after the condition evaluation - val tmp = ArrayList(currentPredecessors) + val tmp = currentPredecessors.toMutableList() val compound = if (node.statement is DoStatement) { createEOG(node.statement) @@ -959,7 +973,7 @@ open class EvaluationOrderGraphPass(ctx: TranslationContext) : TranslationUnitPa } else { node.statement as Block } - currentPredecessors = ArrayList() + currentPredecessors = mutableListOf() for (subStatement in compound.statements) { if (subStatement is CaseStatement || subStatement is DefaultStatement) { currentPredecessors.addAll(tmp) @@ -989,7 +1003,7 @@ open class EvaluationOrderGraphPass(ctx: TranslationContext) : TranslationUnitPa createEOG(node.condition) pushToEOG(node) // To have semantic information after the condition evaluation nextEdgeBranch = true - val tmpEOGNodes = ArrayList(currentPredecessors) + val tmpEOGNodes = currentPredecessors.toMutableList() createEOG(node.statement) connectCurrentToLoopStart() @@ -1020,7 +1034,7 @@ open class EvaluationOrderGraphPass(ctx: TranslationContext) : TranslationUnitPa */ protected fun reachableFromValidEOGRoot(node: Node): Boolean { val passedBy = mutableSetOf() - val workList = ArrayList(node.prevEOG) + val workList = node.prevEOG.toMutableList() while (workList.isNotEmpty()) { val toProcess = workList[0] workList.remove(toProcess) diff --git a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/Python.kt b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/Python.kt index b0fbf27dad..774bd91c0b 100644 --- a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/Python.kt +++ b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/Python.kt @@ -437,7 +437,7 @@ interface Python { "handlers" of pyObject } val orelse: kotlin.collections.List by lazy { "orelse" of pyObject } - val stmt: kotlin.collections.List by lazy { "StmtBase" of pyObject } + val finalbody: kotlin.collections.List by lazy { "finalbody" of pyObject } } /** diff --git a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonLanguage.kt b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonLanguage.kt index 2c79b94ee5..49fd56726f 100644 --- a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonLanguage.kt +++ b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonLanguage.kt @@ -25,11 +25,7 @@ */ package de.fraunhofer.aisec.cpg.frontends.python -import de.fraunhofer.aisec.cpg.frontends.HasFunctionStyleConstruction -import de.fraunhofer.aisec.cpg.frontends.HasOperatorOverloading -import de.fraunhofer.aisec.cpg.frontends.HasShortCircuitOperators -import de.fraunhofer.aisec.cpg.frontends.Language -import de.fraunhofer.aisec.cpg.frontends.of +import de.fraunhofer.aisec.cpg.frontends.* import de.fraunhofer.aisec.cpg.graph.HasOverloadedOperation import de.fraunhofer.aisec.cpg.graph.autoType import de.fraunhofer.aisec.cpg.graph.declarations.ParameterDeclaration diff --git a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt index fdf05aed61..f2de3a3a2d 100644 --- a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt +++ b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt @@ -33,15 +33,8 @@ import de.fraunhofer.aisec.cpg.frontends.python.PythonLanguage.Companion.MODIFIE import de.fraunhofer.aisec.cpg.graph.* import de.fraunhofer.aisec.cpg.graph.Annotation import de.fraunhofer.aisec.cpg.graph.declarations.* -import de.fraunhofer.aisec.cpg.graph.statements.AssertStatement -import de.fraunhofer.aisec.cpg.graph.statements.DeclarationStatement -import de.fraunhofer.aisec.cpg.graph.statements.Statement -import de.fraunhofer.aisec.cpg.graph.statements.expressions.AssignExpression -import de.fraunhofer.aisec.cpg.graph.statements.expressions.Block -import de.fraunhofer.aisec.cpg.graph.statements.expressions.DeleteExpression -import de.fraunhofer.aisec.cpg.graph.statements.expressions.Expression -import de.fraunhofer.aisec.cpg.graph.statements.expressions.MemberExpression -import de.fraunhofer.aisec.cpg.graph.statements.expressions.ProblemExpression +import de.fraunhofer.aisec.cpg.graph.statements.* +import de.fraunhofer.aisec.cpg.graph.statements.expressions.* import de.fraunhofer.aisec.cpg.graph.types.FunctionType import de.fraunhofer.aisec.cpg.helpers.Util import kotlin.collections.plusAssign @@ -69,12 +62,12 @@ class StatementHandler(frontend: PythonLanguageFrontend) : is Python.AST.Break -> newBreakStatement(rawNode = node) is Python.AST.Continue -> newContinueStatement(rawNode = node) is Python.AST.Assert -> handleAssert(node) + is Python.AST.Try -> handleTryStatement(node) is Python.AST.Delete -> handleDelete(node) is Python.AST.Global, is Python.AST.Match, is Python.AST.Nonlocal, is Python.AST.Raise, - is Python.AST.Try, is Python.AST.TryStar, is Python.AST.With, is Python.AST.AsyncWith -> @@ -85,6 +78,57 @@ class StatementHandler(frontend: PythonLanguageFrontend) : } } + /** + * Translates an [`excepthandler`] which can only be a + * [`ExceptHandler`](https://docs.python.org/3/library/ast.html#ast.ExceptHandler) to a + * [CatchClause]. + * + * It adds all the statements to the body and will set a parameter if it exists. For the + * catch-all clause, we do not set the [CatchClause.parameter]. + */ + private fun handleBaseExcepthandler(node: Python.AST.BaseExcepthandler): CatchClause { + return when (node) { + is Python.AST.ExceptHandler -> { + val catchClause = newCatchClause(rawNode = node) + catchClause.body = makeBlock(node.body, node) + // The parameter can have a type but if the type is None/null, it's the "catch-all" + // clause. + // In this case, it also cannot have a name, so we can skip the variable + // declaration. + if (node.type != null) { + // the parameter can have a name, or we use the anonymous identifier _ + catchClause.parameter = + newVariableDeclaration( + name = node.name ?: "", + type = frontend.typeOf(node.type), + rawNode = node + ) + } + catchClause + } + } + } + + /** + * Translates a Python [`Try`](https://docs.python.org/3/library/ast.html#ast.Try) into a + * [TryStatement]. + */ + private fun handleTryStatement(node: Python.AST.Try): TryStatement { + val tryStatement = newTryStatement(rawNode = node) + tryStatement.tryBlock = makeBlock(node.body, node) + tryStatement.catchClauses.addAll(node.handlers.map { handleBaseExcepthandler(it) }) + + if (node.orelse.isNotEmpty()) { + tryStatement.elseBlock = makeBlock(node.orelse, node) + } + + if (node.finalbody.isNotEmpty()) { + tryStatement.finallyBlock = makeBlock(node.finalbody, node) + } + + return tryStatement + } + /** * Translates a Python [`Delete`](https://docs.python.org/3/library/ast.html#ast.Delete) into a * [DeleteExpression]. diff --git a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/statementHandler/StatementHandlerTest.kt b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/statementHandler/StatementHandlerTest.kt index 1023f22fc7..4ea13e479b 100644 --- a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/statementHandler/StatementHandlerTest.kt +++ b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/statementHandler/StatementHandlerTest.kt @@ -28,19 +28,15 @@ package de.fraunhofer.aisec.cpg.frontends.python.statementHandler import de.fraunhofer.aisec.cpg.TranslationResult import de.fraunhofer.aisec.cpg.frontends.python.* import de.fraunhofer.aisec.cpg.graph.* -import de.fraunhofer.aisec.cpg.graph.statements.AssertStatement -import de.fraunhofer.aisec.cpg.graph.statements.expressions.DeleteExpression -import de.fraunhofer.aisec.cpg.graph.statements.expressions.Literal -import de.fraunhofer.aisec.cpg.graph.statements.expressions.SubscriptExpression +import de.fraunhofer.aisec.cpg.graph.statements.* +import de.fraunhofer.aisec.cpg.graph.statements.expressions.* +import de.fraunhofer.aisec.cpg.helpers.Util import de.fraunhofer.aisec.cpg.test.* import de.fraunhofer.aisec.cpg.test.analyze import de.fraunhofer.aisec.cpg.test.analyzeAndGetFirstTU import de.fraunhofer.aisec.cpg.test.assertResolvedType import java.nio.file.Path -import kotlin.test.Test -import kotlin.test.assertEquals -import kotlin.test.assertNotNull -import kotlin.test.assertTrue +import kotlin.test.* import org.junit.jupiter.api.BeforeAll import org.junit.jupiter.api.TestInstance @@ -63,6 +59,69 @@ class StatementHandlerTest : BaseTest() { assertNotNull(result) } + @Test + fun testTry() { + val tu = + analyzeAndGetFirstTU(listOf(topLevel.resolve("try.py").toFile()), topLevel, true) { + it.registerLanguage() + } + assertNotNull(tu) + + val tryAll = tu.functions["tryAll"]?.trys?.singleOrNull() + assertNotNull(tryAll) + + assertEquals(1, tryAll.tryBlock?.statements?.size) + + assertEquals(3, tryAll.catchClauses.size) + assertLocalName("", tryAll.catchClauses[0].parameter) + assertLocalName("e", tryAll.catchClauses[1].parameter) + assertNull(tryAll.catchClauses[2].parameter) + + assertEquals(1, tryAll.elseBlock?.statements?.size) + assertEquals(1, tryAll.finallyBlock?.statements?.size) + + val tryOnlyFinally = tu.functions["tryOnlyFinally"]?.trys?.singleOrNull() + assertNotNull(tryOnlyFinally) + + assertEquals(1, tryOnlyFinally.tryBlock?.statements?.size) + + assertEquals(0, tryOnlyFinally.catchClauses.size) + + assertNull(tryOnlyFinally.elseBlock) + assertEquals(1, tryOnlyFinally.finallyBlock?.statements?.size) + + val tryOnlyExcept = tu.functions["tryOnlyExcept"]?.trys?.singleOrNull() + assertNotNull(tryOnlyExcept) + + assertEquals(1, tryOnlyExcept.tryBlock?.statements?.size) + + assertEquals(1, tryOnlyExcept.catchClauses.size) + assertNull(tryOnlyExcept.catchClauses.single().parameter) + + assertNull(tryOnlyExcept.elseBlock) + assertNull(tryOnlyExcept.finallyBlock) + + // Test EOG integrity with else block + + // All entries to the else block must come from the try block + assertTrue( + Util.eogConnect( + n = tryAll.elseBlock, + en = Util.Edge.ENTRIES, + refs = listOf(tryAll.tryBlock) + ) + ) + + // All exits from the else block must go to the entries of the non-empty finals block + assertTrue( + Util.eogConnect( + n = tryAll.elseBlock, + en = Util.Edge.EXITS, + refs = listOf(tryAll.finallyBlock) + ) + ) + } + @Test fun testAsync() { val tu = diff --git a/cpg-language-python/src/test/resources/python/try.py b/cpg-language-python/src/test/resources/python/try.py new file mode 100644 index 0000000000..dbf91aa85e --- /dev/null +++ b/cpg-language-python/src/test/resources/python/try.py @@ -0,0 +1,25 @@ +def tryAll(a): + try: + b = a+2 + except Exception1: + print("There was an occurrence of Exception1") + except OtherException as e: + print("We saw exception" + e) + except: + print("Catch all!") + else: + print("All good, got " + b) + finally: + print("It's over") + +def tryOnlyFinally(a): + try: + b = a+2 + finally: + print("It's over") + +def tryOnlyExcept(a): + try: + b = a+2 + except: + print("Fail") \ No newline at end of file