From 8b9b2973565ef03131eda2f4c05ae50a9c78f116 Mon Sep 17 00:00:00 2001 From: Christian Banse Date: Wed, 9 Aug 2023 10:04:09 +0200 Subject: [PATCH] Overhaul of type propagation (#1268) Co-authored-by: Alexander Kuechler Co-authored-by: Maximilian Kaul --- .../aisec/cpg/analysis/MultiValueEvaluator.kt | 106 +++-- .../aisec/cpg/analysis/ValueEvaluator.kt | 53 ++- .../cpg/analysis/MultiValueEvaluatorTest.kt | 7 +- .../aisec/cpg/analysis/SizeEvaluatorTest.kt | 8 +- .../aisec/cpg/console/Extensions.kt | 2 +- .../de/fraunhofer/aisec/cpg/ScopeManager.kt | 40 +- .../aisec/cpg/TranslationConfiguration.kt | 34 +- .../aisec/cpg/TranslationManager.kt | 15 - .../de/fraunhofer/aisec/cpg/TypeManager.kt | 28 -- .../cpg/frontends/CompilationDatabase.kt | 11 + .../fraunhofer/aisec/cpg/frontends/Handler.kt | 11 + .../aisec/cpg/frontends/Language.kt | 51 ++- .../fraunhofer/aisec/cpg/graph/Assignment.kt | 1 + .../aisec/cpg/graph/ExpressionBuilder.kt | 7 +- .../fraunhofer/aisec/cpg/graph/Extensions.kt | 5 + .../de/fraunhofer/aisec/cpg/graph/HasBase.kt | 4 +- .../aisec/cpg/graph/HasInitializer.kt | 1 + .../aisec/cpg/graph/HasOperatorCode.kt | 35 ++ .../de/fraunhofer/aisec/cpg/graph/Name.kt | 5 +- .../de/fraunhofer/aisec/cpg/graph/Node.kt | 4 +- .../fraunhofer/aisec/cpg/graph/TypeBuilder.kt | 4 + .../aisec/cpg/graph/builder/Fluent.kt | 146 ++++++- .../graph/declarations/FieldDeclaration.kt | 73 +--- .../graph/declarations/FunctionDeclaration.kt | 1 + .../declarations/TypeParamDeclaration.kt | 6 +- .../graph/declarations/ValueDeclaration.kt | 194 ++------- .../graph/declarations/VariableDeclaration.kt | 94 ++--- .../expressions/ArrayCreationExpression.kt | 27 +- .../ArraySubscriptionExpression.kt | 54 ++- .../expressions/AssignExpression.kt | 131 ++++--- .../statements/expressions/BinaryOperator.kt | 143 ++----- .../statements/expressions/CallExpression.kt | 46 +-- .../statements/expressions/CastExpression.kt | 63 +-- .../expressions/ConditionalExpression.kt | 63 ++- .../expressions/ConstructExpression.kt | 44 +-- .../DeclaredReferenceExpression.kt | 103 ++--- .../statements/expressions/Expression.kt | 199 +--------- .../statements/expressions/ExpressionList.kt | 52 +-- .../expressions/InitializerListExpression.kt | 97 +++-- .../expressions/LambdaExpression.kt | 52 +-- .../expressions/MemberCallExpression.kt | 2 +- .../expressions/MemberExpression.kt | 32 +- .../statements/expressions/UnaryOperator.kt | 132 ++----- .../aisec/cpg/graph/types/AutoType.kt | 50 +++ .../aisec/cpg/graph/types/FunctionType.kt | 1 - .../aisec/cpg/graph/types/HasType.kt | 210 ++++++---- .../aisec/cpg/graph/types/ObjectType.kt | 1 - .../cpg/graph/types/SecondaryTypeEdge.kt | 39 ++ .../fraunhofer/aisec/cpg/graph/types/Type.kt | 2 + .../aisec/cpg/graph/types/UnknownType.kt | 6 +- .../aisec/cpg/helpers/SubgraphWalker.kt | 16 - .../aisec/cpg/passes/CXXCallResolverHelper.kt | 45 +-- .../aisec/cpg/passes/CallResolver.kt | 6 +- .../cpg/passes/ControlFlowSensitiveDFGPass.kt | 48 ++- .../de/fraunhofer/aisec/cpg/passes/DFGPass.kt | 9 +- .../aisec/cpg/passes/TypeHierarchyResolver.kt | 3 - .../aisec/cpg/passes/TypeResolver.kt | 32 +- .../aisec/cpg/passes/VariableUsageResolver.kt | 48 ++- .../aisec/cpg/passes/inference/Inference.kt | 57 ++- .../de/fraunhofer/aisec/cpg/GraphExamples.kt | 4 +- .../aisec/cpg/enhancements/InferenceTest.kt | 51 +-- .../fraunhofer/aisec/cpg/graph/FluentTest.kt | 18 +- .../aisec/cpg/graph/ShortcutsTest.kt | 81 ++-- .../expressions/AssignExpressionTest.kt | 65 ++-- .../cpg/graph/types/TypePropagationTest.kt | 368 ++++++++++++++++-- .../aisec/cpg/helpers/SubgraphWalkerTest.kt | 1 - .../de/fraunhofer/aisec/cpg/passes/DFGTest.kt | 4 +- .../de/fraunhofer/aisec/cpg/TestUtils.kt | 1 - .../aisec/cpg/frontends/cxx/CLanguage.kt | 18 +- .../aisec/cpg/frontends/cxx/CPPLanguage.kt | 19 +- .../aisec/cpg/frontends/cxx/CXXHandler.kt | 3 + .../cpg/frontends/cxx/CXXLanguageFrontend.kt | 45 ++- .../cpg/frontends/cxx/DeclarationHandler.kt | 242 +++++++----- .../cpg/frontends/cxx/DeclaratorHandler.kt | 23 +- .../cpg/frontends/cxx/ExpressionHandler.kt | 112 +++--- .../cpg/frontends/cxx/InitializerHandler.kt | 36 +- .../aisec/cpg/PerformanceRegressionTest.kt | 1 - .../enhancements/calls/FunctionPointerTest.kt | 4 +- .../templates/FunctionTemplateTest.kt | 28 +- .../aisec/cpg/enhancements/types/TypeTests.kt | 6 +- .../frontends/cxx/CXXLanguageFrontendTest.kt | 170 ++++---- .../aisec/cpg/frontends/cxx/LambdaTest.kt | 27 +- .../aisec/cpg/helpers/BenchmarkCXXTest.kt | 4 +- .../{ => cxx}/assignmentexpression.cpp | 0 .../resources/{ => cxx}/binaryoperator.cpp | 0 .../{components => cxx}/castexpr.cpp | 0 .../src/test/resources/cxx/fancy_types.cpp | 3 + .../{components => cxx}/foreachstmt.cpp | 0 .../cpg/frontends/golang/ExpressionHandler.kt | 4 +- .../aisec/cpg/frontends/golang/GoHandler.kt | 2 + .../frontends/golang/SpecificationHandler.kt | 2 + .../aisec/cpg/passes/GoExtraPass.kt | 13 +- .../src/test/resources/golang/literal.go | 5 +- .../cpg/frontends/java/ExpressionHandler.kt | 31 +- .../cpg/frontends/java/StatementHandler.kt | 4 +- .../cpg/passes/JavaCallResolverHelper.kt | 16 +- .../java/JavaLanguageFrontendTest.kt | 10 +- .../aisec/cpg/graph/types/TypeTests.kt | 4 +- .../cpg/frontends/llvm/ExpressionHandler.kt | 3 +- .../cpg/frontends/llvm/StatementHandler.kt | 6 +- .../cpg/frontends/python/PythonLanguage.kt | 13 +- .../src/main/python/CPGPython/_expressions.py | 12 +- .../src/main/python/CPGPython/_statements.py | 47 ++- .../frontends/python/PythonFrontendTest.kt | 42 +- .../frontends/typescript/ExpressionHandler.kt | 2 +- .../src/main/nodejs/package-lock.json | 2 +- .../aisec/cpg_vis_neo4j/ApplicationTest.kt | 4 +- 107 files changed, 2229 insertions(+), 2086 deletions(-) create mode 100644 cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/HasOperatorCode.kt create mode 100644 cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/AutoType.kt create mode 100644 cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/SecondaryTypeEdge.kt rename cpg-language-cxx/src/test/resources/{ => cxx}/assignmentexpression.cpp (100%) rename cpg-language-cxx/src/test/resources/{ => cxx}/binaryoperator.cpp (100%) rename cpg-language-cxx/src/test/resources/{components => cxx}/castexpr.cpp (100%) create mode 100644 cpg-language-cxx/src/test/resources/cxx/fancy_types.cpp rename cpg-language-cxx/src/test/resources/{components => cxx}/foreachstmt.cpp (100%) diff --git a/cpg-analysis/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/MultiValueEvaluator.kt b/cpg-analysis/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/MultiValueEvaluator.kt index 8760d1e5b2..c649aeecb7 100644 --- a/cpg-analysis/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/MultiValueEvaluator.kt +++ b/cpg-analysis/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/MultiValueEvaluator.kt @@ -28,6 +28,7 @@ package de.fraunhofer.aisec.cpg.analysis import de.fraunhofer.aisec.cpg.graph.Node import de.fraunhofer.aisec.cpg.graph.declarations.FieldDeclaration import de.fraunhofer.aisec.cpg.graph.declarations.VariableDeclaration +import de.fraunhofer.aisec.cpg.graph.invoke import de.fraunhofer.aisec.cpg.graph.statements.DeclarationStatement import de.fraunhofer.aisec.cpg.graph.statements.ForStatement import de.fraunhofer.aisec.cpg.graph.statements.expressions.* @@ -78,6 +79,7 @@ class MultiValueEvaluator : ValueEvaluator() { is Literal<*> -> return node.value is DeclaredReferenceExpression -> return handleDeclaredReferenceExpression(node, depth) is UnaryOperator -> return handleUnaryOp(node, depth) + is AssignExpression -> return handleAssignExpression(node, depth) is BinaryOperator -> return handleBinaryOperator(node, depth) // Casts are just a wrapper in this case, we are interested in the inner expression is CastExpression -> return this.evaluateInternal(node.expression, depth + 1) @@ -92,6 +94,48 @@ class MultiValueEvaluator : ValueEvaluator() { return cannotEvaluate(node, this) } + /** + * We are handling some basic arithmetic compound assignment operations and string operations + * that are more or less language-independent. + */ + override fun handleAssignExpression(node: AssignExpression, depth: Int): Any? { + // This only works for compound assignments + if (!node.isCompoundAssignment) { + return super.handleAssignExpression(node, depth) + } + + // Resolve lhs + val lhsValue = evaluateInternal(node.lhs.singleOrNull(), depth + 1) + // Resolve rhs + val rhsValue = evaluateInternal(node.rhs.singleOrNull(), depth + 1) + + if (lhsValue !is Collection<*> && rhsValue !is Collection<*>) { + return computeBinaryOpEffect(lhsValue, rhsValue, node) + } + + val result = mutableSetOf() + if (lhsValue is Collection<*>) { + // lhsValue is a collection. We compute the result for all lhsValues with all the + // rhsValue(s). + for (lhs in lhsValue) { + if (rhsValue is Collection<*>) { + result.addAll(rhsValue.map { r -> computeBinaryOpEffect(lhs, r, node) }) + } else { + result.add(computeBinaryOpEffect(lhs, rhsValue, node)) + } + } + } else { + // lhsValue is not a collection (so rhsValues is because if both wouldn't be a + // collection, this would be covered by the if-statement some lines above). We compute + // the result for the lhsValue with all the rhsValues. + result.addAll( + (rhsValue as Collection<*>).map { r -> computeBinaryOpEffect(lhsValue, r, node) } + ) + } + + return result + } + /** * We are handling some basic arithmetic binary operations and string operations that are more * or less language-independent. @@ -279,60 +323,64 @@ class MultiValueEvaluator : ValueEvaluator() { val loopOp = loop.iterationStatement loopVar = when (loopOp) { - is BinaryOperator -> { + is AssignExpression -> { if ( loopOp.operatorCode == "=" && - (loopOp.lhs as? DeclaredReferenceExpression)?.refersTo == - expr.refersTo && - loopOp.rhs is BinaryOperator + (loopOp.lhs.singleOrNull() as? DeclaredReferenceExpression) + ?.refersTo == expr.refersTo && + loopOp.rhs.singleOrNull() is BinaryOperator ) { // Assignment to the variable, take the rhs and see if it's also a // binary operator val opLhs = if ( - ((loopOp.rhs as BinaryOperator).lhs + ((loopOp.rhs())?.lhs as? DeclaredReferenceExpression) ?.refersTo == expr.refersTo ) { loopVar } else { - (loopOp.rhs as BinaryOperator).lhs + (loopOp.rhs())?.lhs } val opRhs = if ( - ((loopOp.rhs as BinaryOperator).rhs + ((loopOp.rhs())?.rhs as? DeclaredReferenceExpression) ?.refersTo == expr.refersTo ) { loopVar } else { - evaluateInternal((loopOp.rhs as BinaryOperator).rhs, depth + 1) + evaluateInternal((loopOp.rhs())?.rhs, depth + 1) } - computeBinaryOpEffect(opLhs, opRhs, (loopOp.rhs as BinaryOperator)) + computeBinaryOpEffect(opLhs, opRhs, (loopOp.rhs())) as? Number } else { - // No idea what this is but it's a binary op... - val opLhs = - if ( - (loopOp.lhs as? DeclaredReferenceExpression)?.refersTo == - expr.refersTo - ) { - loopVar - } else { - loopOp.lhs - } - val opRhs = - if ( - (loopOp.rhs as? DeclaredReferenceExpression)?.refersTo == - expr.refersTo - ) { - loopVar - } else { - loopOp.rhs - } - computeBinaryOpEffect(opLhs, opRhs, loopOp) as? Number + cannotEvaluate(loopOp, this) } } + is BinaryOperator -> { + + // No idea what this is but it's a binary op... + val opLhs = + if ( + (loopOp.lhs as? DeclaredReferenceExpression)?.refersTo == + expr.refersTo + ) { + loopVar + } else { + loopOp.lhs + } + val opRhs = + if ( + (loopOp.rhs as? DeclaredReferenceExpression)?.refersTo == + expr.refersTo + ) { + loopVar + } else { + loopOp.rhs + } + computeBinaryOpEffect(opLhs, opRhs, loopOp) as? Number + } is UnaryOperator -> { computeUnaryOpEffect( if ( diff --git a/cpg-analysis/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/ValueEvaluator.kt b/cpg-analysis/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/ValueEvaluator.kt index 5e1d27d582..5fa5f49537 100644 --- a/cpg-analysis/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/ValueEvaluator.kt +++ b/cpg-analysis/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/ValueEvaluator.kt @@ -26,6 +26,7 @@ package de.fraunhofer.aisec.cpg.analysis import de.fraunhofer.aisec.cpg.graph.AccessValues +import de.fraunhofer.aisec.cpg.graph.HasOperatorCode import de.fraunhofer.aisec.cpg.graph.Node import de.fraunhofer.aisec.cpg.graph.declarations.VariableDeclaration import de.fraunhofer.aisec.cpg.graph.statements.expressions.* @@ -99,7 +100,7 @@ open class ValueEvaluator( // While we are not handling different paths of variables with If statements, we can // easily be partly path-sensitive in a conditional expression is ConditionalExpression -> return handleConditionalExpression(node, depth) - is AssignExpression -> return handleAssignExpression(node) + is AssignExpression -> return handleAssignExpression(node, depth) } // At this point, we cannot evaluate, and we are calling our [cannotEvaluate] hook, maybe @@ -108,8 +109,19 @@ open class ValueEvaluator( } /** Under certain circumstances, an assignment can also be used as an expression. */ - private fun handleAssignExpression(node: AssignExpression): Any? { - if (node.usedAsExpression) { + protected open fun handleAssignExpression(node: AssignExpression, depth: Int): Any? { + // Handle compound assignments. Only possible with single values + val lhs = node.lhs.singleOrNull() + val rhs = node.rhs.singleOrNull() + if (lhs != null && rhs != null && node.isCompoundAssignment) { + // Resolve rhs + val rhsValue = evaluateInternal(rhs, depth + 1) + + // Resolve lhs + val lhsValue = evaluateInternal(lhs, depth + 1) + + return computeBinaryOpEffect(lhsValue, rhsValue, node) + } else if (node.usedAsExpression) { return node.expressionValue } @@ -130,12 +142,19 @@ open class ValueEvaluator( return computeBinaryOpEffect(lhsValue, rhsValue, expr) } + /** + * Computes the effect of basic "binary" operators. + * + * Note: this is both used by a [BinaryOperator] with basic arithmetic operations as well as + * [AssignExpression], if [AssignExpression.isCompoundAssignment] is true. + */ protected fun computeBinaryOpEffect( lhsValue: Any?, rhsValue: Any?, - expr: BinaryOperator + has: HasOperatorCode?, ): Any? { - return when (expr.operatorCode) { + val expr = has as? Expression + return when (has?.operatorCode) { "+", "+=" -> handlePlus(lhsValue, rhsValue, expr) "-", @@ -149,11 +168,11 @@ open class ValueEvaluator( "<" -> handleLess(lhsValue, rhsValue, expr) "<=" -> handleLEq(lhsValue, rhsValue, expr) "==" -> handleEq(lhsValue, rhsValue, expr) - else -> cannotEvaluate(expr, this) + else -> cannotEvaluate(expr as Node, this) } } - private fun handlePlus(lhsValue: Any?, rhsValue: Any?, expr: BinaryOperator): Any? { + private fun handlePlus(lhsValue: Any?, rhsValue: Any?, expr: Expression?): Any? { return when { lhsValue is String -> lhsValue + rhsValue lhsValue is Int && (rhsValue is Double || rhsValue is Float) -> @@ -174,7 +193,7 @@ open class ValueEvaluator( } } - private fun handleMinus(lhsValue: Any?, rhsValue: Any?, expr: BinaryOperator): Any? { + private fun handleMinus(lhsValue: Any?, rhsValue: Any?, expr: Expression?): Any? { return when { lhsValue is Int && (rhsValue is Double || rhsValue is Float) -> lhsValue - (rhsValue as Number).toDouble() @@ -194,7 +213,7 @@ open class ValueEvaluator( } } - private fun handleDiv(lhsValue: Any?, rhsValue: Any?, expr: BinaryOperator): Any? { + private fun handleDiv(lhsValue: Any?, rhsValue: Any?, expr: Expression?): Any? { return when { rhsValue == 0 -> cannotEvaluate(expr, this) lhsValue is Int && (rhsValue is Double || rhsValue is Float) -> @@ -215,7 +234,7 @@ open class ValueEvaluator( } } - private fun handleTimes(lhsValue: Any?, rhsValue: Any?, expr: BinaryOperator): Any? { + private fun handleTimes(lhsValue: Any?, rhsValue: Any?, expr: Expression?): Any? { return when { lhsValue is Int && (rhsValue is Double || rhsValue is Float) -> lhsValue * (rhsValue as Number).toDouble() @@ -235,7 +254,7 @@ open class ValueEvaluator( } } - private fun handleGreater(lhsValue: Any?, rhsValue: Any?, expr: BinaryOperator): Any? { + private fun handleGreater(lhsValue: Any?, rhsValue: Any?, expr: Expression?): Any? { return if (lhsValue is Number && rhsValue is Number) { lhsValue.compareTo(rhsValue) > 0 } else { @@ -243,7 +262,7 @@ open class ValueEvaluator( } } - private fun handleGEq(lhsValue: Any?, rhsValue: Any?, expr: BinaryOperator): Any? { + private fun handleGEq(lhsValue: Any?, rhsValue: Any?, expr: Expression?): Any? { return if (lhsValue is Number && rhsValue is Number) { lhsValue.compareTo(rhsValue) >= 0 } else { @@ -251,7 +270,7 @@ open class ValueEvaluator( } } - private fun handleLess(lhsValue: Any?, rhsValue: Any?, expr: BinaryOperator): Any? { + private fun handleLess(lhsValue: Any?, rhsValue: Any?, expr: Expression?): Any? { return if (lhsValue is Number && rhsValue is Number) { lhsValue.compareTo(rhsValue) < 0 } else { @@ -259,7 +278,7 @@ open class ValueEvaluator( } } - private fun handleLEq(lhsValue: Any?, rhsValue: Any?, expr: BinaryOperator): Any? { + private fun handleLEq(lhsValue: Any?, rhsValue: Any?, expr: Expression?): Any? { return if (lhsValue is Number && rhsValue is Number) { lhsValue.compareTo(rhsValue) <= 0 } else { @@ -267,7 +286,7 @@ open class ValueEvaluator( } } - private fun handleEq(lhsValue: Any?, rhsValue: Any?, expr: BinaryOperator): Any? { + private fun handleEq(lhsValue: Any?, rhsValue: Any?, expr: Expression?): Any? { return if (lhsValue is Number && rhsValue is Number) { lhsValue.compareTo(rhsValue) == 0 } else { @@ -414,14 +433,14 @@ open class ValueEvaluator( // Remove the self reference list = list.filter { - !((it is BinaryOperator && it.lhs == ref) || + !((it is AssignExpression && it.lhs.singleOrNull() == ref) || (it is UnaryOperator && it.input == ref)) } } else if (ref.access == AccessValues.READWRITE && !isCase2) { // Consider only the self reference list = list.filter { - ((it is BinaryOperator && it.lhs == ref) || + ((it is AssignExpression && it.lhs.singleOrNull() == ref) || (it is UnaryOperator && it.input == ref)) } } diff --git a/cpg-analysis/src/test/kotlin/de/fraunhofer/aisec/cpg/analysis/MultiValueEvaluatorTest.kt b/cpg-analysis/src/test/kotlin/de/fraunhofer/aisec/cpg/analysis/MultiValueEvaluatorTest.kt index f401058bde..099ca6255c 100644 --- a/cpg-analysis/src/test/kotlin/de/fraunhofer/aisec/cpg/analysis/MultiValueEvaluatorTest.kt +++ b/cpg-analysis/src/test/kotlin/de/fraunhofer/aisec/cpg/analysis/MultiValueEvaluatorTest.kt @@ -33,7 +33,7 @@ import de.fraunhofer.aisec.cpg.graph.evaluate import de.fraunhofer.aisec.cpg.graph.statements.CompoundStatement import de.fraunhofer.aisec.cpg.graph.statements.DeclarationStatement import de.fraunhofer.aisec.cpg.graph.statements.ForStatement -import de.fraunhofer.aisec.cpg.graph.statements.expressions.BinaryOperator +import de.fraunhofer.aisec.cpg.graph.statements.expressions.AssignExpression import de.fraunhofer.aisec.cpg.graph.statements.expressions.CallExpression import de.fraunhofer.aisec.cpg.passes.EdgeCachePass import java.nio.file.Path @@ -211,7 +211,10 @@ class MultiValueEvaluatorTest { assertNotNull(forLoop) val evaluator = MultiValueEvaluator() - val iVar = ((forLoop.statement as CompoundStatement).statements[0] as BinaryOperator).rhs + val iVarList = + ((forLoop.statement as CompoundStatement).statements[0] as AssignExpression).rhs + assertEquals(1, iVarList.size) + val iVar = iVarList.first() val value = evaluator.evaluate(iVar) as ConcreteNumberSet assertEquals(setOf(0, 1, 2, 3, 4, 5), value.values) } diff --git a/cpg-analysis/src/test/kotlin/de/fraunhofer/aisec/cpg/analysis/SizeEvaluatorTest.kt b/cpg-analysis/src/test/kotlin/de/fraunhofer/aisec/cpg/analysis/SizeEvaluatorTest.kt index c7ee6bb57e..2f470c3695 100644 --- a/cpg-analysis/src/test/kotlin/de/fraunhofer/aisec/cpg/analysis/SizeEvaluatorTest.kt +++ b/cpg-analysis/src/test/kotlin/de/fraunhofer/aisec/cpg/analysis/SizeEvaluatorTest.kt @@ -31,10 +31,12 @@ import de.fraunhofer.aisec.cpg.graph.bodyOrNull import de.fraunhofer.aisec.cpg.graph.byNameOrNull import de.fraunhofer.aisec.cpg.graph.declarations.MethodDeclaration import de.fraunhofer.aisec.cpg.graph.declarations.RecordDeclaration +import de.fraunhofer.aisec.cpg.graph.invoke import de.fraunhofer.aisec.cpg.graph.statements.CompoundStatement import de.fraunhofer.aisec.cpg.graph.statements.DeclarationStatement import de.fraunhofer.aisec.cpg.graph.statements.ForStatement -import de.fraunhofer.aisec.cpg.graph.statements.expressions.BinaryOperator +import de.fraunhofer.aisec.cpg.graph.statements.expressions.ArraySubscriptionExpression +import de.fraunhofer.aisec.cpg.graph.statements.expressions.AssignExpression import de.fraunhofer.aisec.cpg.graph.statements.expressions.CallExpression import java.nio.file.Path import kotlin.test.Test @@ -105,7 +107,9 @@ class SizeEvaluatorTest { assertNotNull(forLoop) val subscriptExpr = - ((forLoop.statement as CompoundStatement).statements[0] as BinaryOperator).lhs + ((forLoop.statement as CompoundStatement).statements[0] as AssignExpression).lhs< + ArraySubscriptionExpression + >() value = evaluator.evaluate(subscriptExpr) as Int assertEquals(3, value) diff --git a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/Extensions.kt b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/Extensions.kt index a1c6177590..274b25f3dc 100644 --- a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/Extensions.kt +++ b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/Extensions.kt @@ -25,7 +25,6 @@ */ package de.fraunhofer.aisec.cpg.console -import de.fraunhofer.aisec.cpg.graph.HasType import de.fraunhofer.aisec.cpg.graph.Node import de.fraunhofer.aisec.cpg.graph.declarations.FunctionDeclaration import de.fraunhofer.aisec.cpg.graph.declarations.VariableDeclaration @@ -33,6 +32,7 @@ import de.fraunhofer.aisec.cpg.graph.statements.CompoundStatement import de.fraunhofer.aisec.cpg.graph.statements.DeclarationStatement import de.fraunhofer.aisec.cpg.graph.statements.IfStatement import de.fraunhofer.aisec.cpg.graph.statements.expressions.* +import de.fraunhofer.aisec.cpg.graph.types.HasType import de.fraunhofer.aisec.cpg.sarif.PhysicalLocation import de.fraunhofer.aisec.cpg.sarif.Region import java.io.File diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/ScopeManager.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/ScopeManager.kt index f1a6fed996..7ab83557f5 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/ScopeManager.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/ScopeManager.kt @@ -36,10 +36,7 @@ import de.fraunhofer.aisec.cpg.graph.types.FunctionPointerType import de.fraunhofer.aisec.cpg.graph.types.IncompleteType import de.fraunhofer.aisec.cpg.graph.types.Type import de.fraunhofer.aisec.cpg.helpers.Util -import de.fraunhofer.aisec.cpg.processing.IVisitor -import de.fraunhofer.aisec.cpg.processing.strategy.Strategy import java.util.* -import java.util.concurrent.atomic.AtomicInteger import java.util.function.Predicate import org.slf4j.LoggerFactory @@ -462,7 +459,9 @@ class ScopeManager : ScopeProvider { if (breakStatement.label == null) { val scope = firstScopeOrNull { scope: Scope? -> scope?.isBreakable() == true } if (scope == null) { - LOGGER.error( + Util.errorWithFileLocation( + breakStatement, + LOGGER, "Break inside of unbreakable scope. The break will be ignored, but may lead " + "to an incorrect graph. The source code is not valid or incomplete." ) @@ -605,10 +604,11 @@ class ScopeManager : ScopeProvider { if ( it.name.lastPartsMatch(ref.name) ) { // TODO: This place is likely to make things fail + var helper = ref.resolutionHelper // If the reference seems to point to a function the entire signature is checked // for equality - if (ref.type is FunctionPointerType && it is FunctionDeclaration) { - val fptrType = (ref as HasType).type as FunctionPointerType + if (helper?.type is FunctionPointerType && it is FunctionDeclaration) { + val fptrType = helper.type as FunctionPointerType // TODO(oxisto): This is the third place where function pointers are // resolved. WHY? // TODO(oxisto): Support multiple return values @@ -793,32 +793,4 @@ class ScopeManager : ScopeProvider { /** Returns the current scope for the [ScopeProvider] interface. */ override val scope: Scope? get() = currentScope - - fun activateTypes(node: Node, typeManager: TypeManager) { - val num = AtomicInteger() - val typeCache = typeManager.typeCache - node.accept( - Strategy::AST_FORWARD, - object : IVisitor() { - override fun visit(t: Node) { - if (t is HasType) { - val typeNode = t as HasType - typeCache.getOrDefault(typeNode, emptyList()).forEach { - (t as HasType).type = - typeManager.resolvePossibleTypedef(it, this@ScopeManager) - } - typeCache.remove(t as HasType) - num.getAndIncrement() - } - } - } - ) - LOGGER.debug("Activated {} nodes for {}", num, node.name) - - // For some nodes it may happen that they are not reachable via AST, but we still need to - // set their type to the requested value - typeCache.forEach { (n, types) -> - types.forEach { t -> n.type = typeManager.resolvePossibleTypedef(t, this) } - } - } } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationConfiguration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationConfiguration.kt index f421af0141..2f051ba193 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationConfiguration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationConfiguration.kt @@ -105,7 +105,6 @@ private constructor( disableCleanup: Boolean, useUnityBuild: Boolean, useParallelFrontends: Boolean, - typeSystemActiveInFrontend: Boolean, inferenceConfiguration: InferenceConfiguration, compilationDatabase: CompilationDatabase?, matchCommentsToNodes: Boolean, @@ -141,13 +140,6 @@ private constructor( */ val useParallelFrontends: Boolean - /** - * If false, the type listener system is only activated once the frontends are done building the - * initial AST structure. This avoids errors where the type of a node may depend on the order in - * which the source files have been parsed. - */ - val typeSystemActiveInFrontend: Boolean - /** * This is the data structure for storing the compilation database. It stores a mapping from the * File to the list of files that have to be included to their path, specified by the parameter @@ -182,7 +174,6 @@ private constructor( this.disableCleanup = disableCleanup this.useUnityBuild = useUnityBuild this.useParallelFrontends = useParallelFrontends - this.typeSystemActiveInFrontend = typeSystemActiveInFrontend this.inferenceConfiguration = inferenceConfiguration this.compilationDatabase = compilationDatabase this.matchCommentsToNodes = matchCommentsToNodes @@ -232,7 +223,6 @@ private constructor( private var disableCleanup = false private var useUnityBuild = false private var useParallelFrontends = false - private var typeSystemActiveInFrontend = true private var inferenceConfiguration = InferenceConfiguration.Builder().build() private var compilationDatabase: CompilationDatabase? = null private var matchCommentsToNodes = false @@ -579,9 +569,7 @@ private constructor( /** * If true, the ASTs for the source files are parsed in parallel, but the passes afterwards * will still run in a single thread. This speeds up initial parsing but makes sure that - * further graph enrichment algorithms remain correct. Please make sure to also set - * [ ][.typeSystemActiveInFrontend] to false to avoid probabilistic errors that appear - * depending on the parsing order. + * further graph enrichment algorithms remain correct. * * @param b the new value */ @@ -590,18 +578,6 @@ private constructor( return this } - /** - * If false, the type system is only activated once the frontends are done building the - * initial AST structure. This avoids errors where the type of a node may depend on the - * order in which the source files have been parsed. - * - * @param b the new value - */ - fun typeSystemActiveInFrontend(b: Boolean): Builder { - typeSystemActiveInFrontend = b - return this - } - fun inferenceConfiguration(configuration: InferenceConfiguration): Builder { inferenceConfiguration = configuration return this @@ -609,13 +585,6 @@ private constructor( @Throws(ConfigurationException::class) fun build(): TranslationConfiguration { - if (useParallelFrontends && typeSystemActiveInFrontend) { - log.warn( - "Not disabling the type system during the frontend " + - "phase is not recommended when using the parallel frontends feature! " + - "This may result in erroneous results." - ) - } registerExtraFrontendPasses() registerReplacedPasses() return TranslationConfiguration( @@ -636,7 +605,6 @@ private constructor( disableCleanup, useUnityBuild, useParallelFrontends, - typeSystemActiveInFrontend, inferenceConfiguration, compilationDatabase, matchCommentsToNodes, diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationManager.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationManager.kt index 9e92b11164..dee1e5ac01 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationManager.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationManager.kt @@ -219,8 +219,6 @@ private constructor( sourceLocations = list } - TypeManager.isTypeSystemActive = ctx.config.typeSystemActiveInFrontend - usedFrontends.addAll( if (useParallelFrontends) { parseParallel(component, result, ctx, sourceLocations) @@ -228,19 +226,6 @@ private constructor( parseSequentially(component, result, ctx, sourceLocations) } ) - - if (!config.typeSystemActiveInFrontend) { - TypeManager.isTypeSystemActive = true - - result.components.forEach { s -> - s.translationUnits.forEach { - val bench = - Benchmark(this.javaClass, "Activating types for ${it.name}", true) - ctx.scopeManager.activateTypes(it, ctx.typeManager) - bench.stop() - } - } - } } return usedFrontends diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TypeManager.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TypeManager.kt index ffed50cca9..0081741eb8 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TypeManager.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TypeManager.kt @@ -45,9 +45,6 @@ import org.apache.commons.lang3.builder.ToStringBuilder import org.slf4j.LoggerFactory class TypeManager { - val typeCache: MutableMap> = - Collections.synchronizedMap(IdentityHashMap()) - private val typeToRecord = Collections.synchronizedMap(HashMap()) /** @@ -213,16 +210,6 @@ class TypeManager { return firstOrderTypes.stream().anyMatch { type: Type -> type.root.name.toString() == name } } - @Synchronized - fun cacheType(node: HasType, type: Type) { - if (!isUnknown(type)) { - val types = typeCache.computeIfAbsent(node) { mutableListOf() } - if (!types.contains(type)) { - types.add(type) - } - } - } - fun isUnknown(type: Type?): Boolean { return type is UnknownType } @@ -240,19 +227,6 @@ class TypeManager { return false } - /** - * @param type oldType that we want to replace - * @param newType newType - * @return true if an objectType with instantiated generics is replaced by the same objectType - * with parameterizedTypes as generics false otherwise - */ - fun stopPropagation(type: Type, newType: Type): Boolean { - return if (type is ObjectType && newType is ObjectType && type.name == newType.name) { - (containsParameterizedType(newType.generics) && - !containsParameterizedType(type.generics)) - } else false - } - private fun rewrapType( type: Type, depth: Int, @@ -637,7 +611,5 @@ class TypeManager { companion object { private val log = LoggerFactory.getLogger(TypeManager::class.java) - - var isTypeSystemActive = true } } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/CompilationDatabase.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/CompilationDatabase.kt index 12125cd394..3f67e857a3 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/CompilationDatabase.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/CompilationDatabase.kt @@ -286,10 +286,21 @@ class CompilationDatabase : ArrayList, HandlerInterface>() private val typeOfT: Class<*>? + /** + * This property contains the last node this handler has successfully processed. It is safe to + * call, even when parsing multiple TUs in parallel, since for each TU, a dedicated + * [LanguageFrontend] is spawned, and for each frontend, a dedicated set of [Handler]s is + * created. Within one TU, the processing is sequential in the AST order. + */ + var lastNode: ResultNode? = null + /** * Searches for a handler matching the most specific superclass of [HandlerNode]. The created * map should thus contain a handler for every semantically different AST node and can reuse @@ -131,7 +139,10 @@ abstract class Handler> : Node() { return when (operation.operatorCode) { "+" -> - if (operation.lhs.propagationType is StringType) { + if (operation.lhs.type is StringType) { // string + anything => string - operation.lhs.propagationType - } else if (operation.rhs.propagationType is StringType) { + operation.lhs.type + } else if (operation.rhs.type is StringType) { // anything + string => string - operation.rhs.propagationType + operation.rhs.type } else { - arithmeticOpTypePropagation( - operation.lhs.propagationType, - operation.rhs.propagationType - ) + arithmeticOpTypePropagation(operation.lhs.type, operation.rhs.type) } "-", "*", - "/" -> - arithmeticOpTypePropagation( - operation.lhs.propagationType, - operation.rhs.propagationType - ) + "/" -> arithmeticOpTypePropagation(operation.lhs.type, operation.rhs.type) "<<", ">>" -> - if ( - operation.lhs.propagationType.isPrimitive && - operation.rhs.propagationType.isPrimitive - ) { + if (operation.lhs.type.isPrimitive && operation.rhs.type.isPrimitive) { // primitive type 1 OP primitive type 2 => primitive type 1 - operation.lhs.propagationType + operation.lhs.type } else { unknownType() } else -> unknownType() // We don't know what is this thing } } + + /** + * When propagating [HasType.assignedTypes] from one node to another, we might want to propagate + * only certain types. A common example is to truncate [NumericType]s, when they are not "big" + * enough. + */ + open fun shouldPropagateType(hasType: HasType, srcType: Type): Boolean { + val node = hasType as Node + var nodeType = hasType.type + + // We only want to add certain types, in case we have a numeric type + if (nodeType is NumericType) { + // We do not allow to propagate non-numeric types into numeric types + return if (srcType !is NumericType) { + false + } else { + val srcWidth = srcType.bitWidth + val lhsWidth = nodeType.bitWidth + // Do not propagate anything if the new type is too big for the current type. + return !(lhsWidth != null && srcWidth != null && lhsWidth < srcWidth) + } + } + + return true + } } /** diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Assignment.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Assignment.kt index 038149efd9..2ef8f01c47 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Assignment.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Assignment.kt @@ -28,6 +28,7 @@ package de.fraunhofer.aisec.cpg.graph import com.fasterxml.jackson.annotation.JsonIgnore import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge import de.fraunhofer.aisec.cpg.graph.statements.expressions.Expression +import de.fraunhofer.aisec.cpg.graph.types.HasType /** An assignment holder is a node that intentionally contains assignment edges. */ interface AssignmentHolder { diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/ExpressionBuilder.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/ExpressionBuilder.kt index 6b401d0521..94eb6f4691 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/ExpressionBuilder.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/ExpressionBuilder.kt @@ -201,10 +201,10 @@ fun MetadataProvider.newConditionalExpression( val node = ConditionalExpression() node.applyMetadata(this, EMPTY_NAME, rawNode, code, true) + node.type = type node.condition = condition node.thenExpr = thenExpr node.elseExpr = elseExpr - node.type = type log(node) return node @@ -531,12 +531,15 @@ fun MetadataProvider.newExpressionList(code: String? = null, rawNode: Any? = nul */ @JvmOverloads fun MetadataProvider.newInitializerListExpression( + targetType: Type = unknownType(), code: String? = null, rawNode: Any? = null ): InitializerListExpression { val node = InitializerListExpression() node.applyMetadata(this, EMPTY_NAME, rawNode, code, true) + node.type = targetType + log(node) return node } @@ -614,10 +617,10 @@ fun Literal.duplicate(implicit: Boolean): Literal { duplicate.language = this.language duplicate.value = this.value duplicate.type = this.type + duplicate.assignedTypes = this.assignedTypes duplicate.code = this.code duplicate.location = this.location duplicate.locals = this.locals - duplicate.possibleSubTypes = this.possibleSubTypes duplicate.argumentIndex = this.argumentIndex duplicate.annotations = this.annotations duplicate.comment = this.comment diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Extensions.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Extensions.kt index a304dbff6e..6c7dbc9f82 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Extensions.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Extensions.kt @@ -572,6 +572,11 @@ val VariableDeclaration.firstAssignment: Expression? ?.value } +/** Returns the [i]-th item in this list (or null) and casts it to [T]. */ +inline operator fun List.invoke(i: Int = 0): T? { + return this.getOrNull(i) as? T +} + operator fun Expression.invoke(): N? { return this as? N } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/HasBase.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/HasBase.kt index 185bc3f22e..d200b24825 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/HasBase.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/HasBase.kt @@ -28,7 +28,7 @@ package de.fraunhofer.aisec.cpg.graph import de.fraunhofer.aisec.cpg.graph.statements.expressions.Expression /** Specifies that a certain node has a base on which it executes an operation. */ -interface HasBase { +interface HasBase : HasOperatorCode { /** The base. */ val base: Expression? @@ -37,5 +37,5 @@ interface HasBase { * The operator that is used to access the base. Usually either `.` or `->`, but some languages * offer additional operator codes. */ - val operatorCode: String? + override val operatorCode: String? } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/HasInitializer.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/HasInitializer.kt index 1bcd1ef01b..ff1c5963cf 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/HasInitializer.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/HasInitializer.kt @@ -26,6 +26,7 @@ package de.fraunhofer.aisec.cpg.graph import de.fraunhofer.aisec.cpg.graph.statements.expressions.Expression +import de.fraunhofer.aisec.cpg.graph.types.HasType /** * Specifies that a certain node has an initializer. It is a special case of [ArgumentHolder], in diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/HasOperatorCode.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/HasOperatorCode.kt new file mode 100644 index 0000000000..68005bbbc9 --- /dev/null +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/HasOperatorCode.kt @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2023, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.graph + +import de.fraunhofer.aisec.cpg.graph.statements.expressions.Expression + +/** A simple interface to denote that the implementing class has some kind of [operatorCode]. */ +interface HasOperatorCode { + + /** The operator code, identifying an operation executed on one or more [Expression]s */ + val operatorCode: String? +} diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Name.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Name.kt index 2e2d52b496..4cb0a1f75c 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Name.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Name.kt @@ -135,10 +135,7 @@ internal fun parseName(fqn: CharSequence, delimiter: String, vararg splitDelimit var name: Name? = null for (part in parts) { - val localName = part.replace(")", "").replace("*", "") - if (localName.isNotEmpty()) { - name = Name(localName, name, delimiter) - } + name = Name(part, name, delimiter) } // Actually this should not occur, but otherwise the compiler won't let us return a diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Node.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Node.kt index 7de7a6996f..62bab149b2 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Node.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Node.kt @@ -294,7 +294,7 @@ open class Node : IVisitable, Persistable, LanguageProvider, ScopeProvider } } - fun addPrevDFG( + open fun addPrevDFG( prev: Node, properties: MutableMap = EnumMap(Properties::class.java) ) { @@ -451,7 +451,7 @@ open class Node : IVisitable, Persistable, LanguageProvider, ScopeProvider companion object { @JvmField var TO_STRING_STYLE: ToStringStyle = ToStringStyle.SHORT_PREFIX_STYLE - protected val log: Logger = LoggerFactory.getLogger(Node::class.java) + @JvmStatic protected val log: Logger = LoggerFactory.getLogger(Node::class.java) const val EMPTY_NAME = "" } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/TypeBuilder.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/TypeBuilder.kt index ca7e9b312c..2e3a70e36a 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/TypeBuilder.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/TypeBuilder.kt @@ -42,6 +42,10 @@ fun MetadataProvider?.unknownType(): Type { } } +fun LanguageProvider.autoType(): Type { + return AutoType(this.language) +} + fun MetadataProvider?.incompleteType(): Type { return IncompleteType() } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/builder/Fluent.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/builder/Fluent.kt index 24af1fcb35..05ce234380 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/builder/Fluent.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/builder/Fluent.kt @@ -32,7 +32,9 @@ import de.fraunhofer.aisec.cpg.graph.declarations.* import de.fraunhofer.aisec.cpg.graph.scopes.RecordScope import de.fraunhofer.aisec.cpg.graph.statements.* import de.fraunhofer.aisec.cpg.graph.statements.expressions.* +import de.fraunhofer.aisec.cpg.graph.types.FunctionType import de.fraunhofer.aisec.cpg.graph.types.Type +import de.fraunhofer.aisec.cpg.graph.types.UnknownType import de.fraunhofer.aisec.cpg.passes.executePassSequential fun LanguageFrontend<*, *>.translationResult( @@ -157,6 +159,9 @@ fun LanguageFrontend<*, *>.function( node.returnTypes = listOf(returnType) } + // Make sure that our function has the correct type + node.type = FunctionType.computeType(node) + scopeManager.enterScope(node) init?.let { it(node) } scopeManager.leaveScope(node) @@ -176,13 +181,16 @@ context(RecordDeclaration) fun LanguageFrontend<*, *>.method( name: CharSequence, returnType: Type = unknownType(), - init: MethodDeclaration.() -> Unit + init: (MethodDeclaration.() -> Unit)? = null ): MethodDeclaration { val node = newMethodDeclaration(name) node.returnTypes = listOf(returnType) + node.type = FunctionType.computeType(node) scopeManager.enterScope(node) - init(node) + if (init != null) { + init(node) + } scopeManager.leaveScope(node) scopeManager.addDeclaration(node) @@ -273,6 +281,26 @@ fun LanguageFrontend<*, *>.returnStmt(init: ReturnStatement.() -> Unit): ReturnS return node } +context(Holder) + +fun LanguageFrontend<*, *>.ase( + init: (ArraySubscriptionExpression.() -> Unit)? = null +): ArraySubscriptionExpression { + val node = newArraySubscriptionExpression() + + if (init != null) { + init(node) + } + + // Only add this to an argument holder if the nearest holder is an argument holder + val holder = this@Holder + if (holder is ArgumentHolder) { + holder += node + } + + return node +} + /** * Creates a new [DeclarationStatement] in the Fluent Node DSL and adds it to the * [StatementHolder.statements] of the nearest enclosing [StatementHolder]. The [init] block can be @@ -331,7 +359,7 @@ fun LanguageFrontend<*, *>.call( init: (CallExpression.() -> Unit)? = null ): CallExpression { // Try to parse the name - val parsedName = parseName(name) + val parsedName = parseName(name, ".") val node = if (parsedName.parent != null) { newMemberCallExpression( @@ -422,6 +450,25 @@ fun LanguageFrontend<*, *>.construct( context(Holder) +fun LanguageFrontend<*, *>.cast( + castType: Type, + init: (CastExpression.() -> Unit)? = null +): CastExpression { + val node = newCastExpression() + node.castType = castType + if (init != null) init(node) + + val holder = this@Holder + if (holder is StatementHolder) { + holder += node + } else if (holder is ArgumentHolder) { + holder += node + } + return node +} + +context(Holder) + fun LanguageFrontend<*, *>.new(init: (NewExpression.() -> Unit)? = null): NewExpression { val node = newNewExpression() if (init != null) init(node) @@ -440,9 +487,11 @@ fun LanguageFrontend<*, *>.memberOrRef(name: Name, type: Type = unknownType()): if (name.parent != null) { newMemberExpression(name.localName, memberOrRef(name.parent)) } else { - newDeclaredReferenceExpression(name.localName, objectType(name.localName)) + newDeclaredReferenceExpression(name.localName) } - node.type = type + if (type !is UnknownType) { + node.type = type + } return node } @@ -760,6 +809,32 @@ fun LanguageFrontend<*, *>.literal(value: N, type: Type = unknownType()): Li return node } +/** + * Creates a new [InitializerListExpression] in the Fluent Node DSL and invokes + * [ArgumentHolder.addArgument] of the nearest enclosing [Holder], but only if it is an + * [ArgumentHolder]. + */ +context(Holder) + +fun LanguageFrontend<*, *>.ile( + targetType: Type = unknownType(), + init: (InitializerListExpression.() -> Unit)? = null +): InitializerListExpression { + val node = newInitializerListExpression(targetType) + + if (init != null) { + init(node) + } + + // Only add this to an argument holder if the nearest holder is an argument holder + val holder = this@Holder + if (holder is ArgumentHolder) { + holder += node + } + + return node +} + /** * Creates a new [DeclaredReferenceExpression] in the Fluent Node DSL and invokes * [ArgumentHolder.addArgument] of the nearest enclosing [Holder], but only if it is an @@ -875,10 +950,8 @@ operator fun Expression.plus(rhs: Expression): BinaryOperator { */ context(LanguageFrontend<*, *>, StatementHolder) -operator fun Expression.plusAssign(rhs: Expression): Unit { - val node = (this@LanguageFrontend).newBinaryOperator("+=") - node.lhs = this - node.rhs = rhs +operator fun Expression.plusAssign(rhs: Expression) { + val node = (this@LanguageFrontend).newAssignExpression("+=", listOf(this), listOf(rhs)) (this@StatementHolder) += node } @@ -1022,7 +1095,7 @@ infix fun Expression.lt(rhs: Expression): BinaryOperator { * Creates a new [ConditionalExpression] with a `=` [BinaryOperator.operatorCode] in the Fluent Node * DSL and invokes [StatementHolder.addStatement] of the nearest enclosing [StatementHolder]. */ -context(LanguageFrontend<*, *>, StatementHolder) +context(LanguageFrontend<*, *>, Holder) fun Expression.conditional( condition: Expression, @@ -1031,7 +1104,11 @@ fun Expression.conditional( ): ConditionalExpression { val node = (this@LanguageFrontend).newConditionalExpression(condition, thenExpr, elseExpr) - (this@StatementHolder) += node + if (this@Holder is StatementHolder) { + (this@Holder) += node + } else if (this@Holder is ArgumentHolder) { + this@Holder += node + } return node } @@ -1042,10 +1119,11 @@ fun Expression.conditional( */ context(LanguageFrontend<*, *>, StatementHolder) -infix fun Expression.assign(init: BinaryOperator.() -> Expression): BinaryOperator { - val node = (this@LanguageFrontend).newBinaryOperator("=") - node.lhs = this - node.rhs = init(node) +infix fun Expression.assign(init: AssignExpression.() -> Expression): AssignExpression { + val node = (this@LanguageFrontend).newAssignExpression("=") + node.lhs = listOf(this) + init(node) + // node.rhs = listOf(init(node)) (this@StatementHolder) += node @@ -1053,15 +1131,13 @@ infix fun Expression.assign(init: BinaryOperator.() -> Expression): BinaryOperat } /** - * Creates a new [BinaryOperator] with a `=` [BinaryOperator.operatorCode] in the Fluent Node DSL - * and invokes [StatementHolder.addStatement] of the nearest enclosing [StatementHolder]. + * Creates a new [AssignExpression] with a `=` [AssignExpression.operatorCode] in the Fluent Node + * DSL and invokes [StatementHolder.addStatement] of the nearest enclosing [StatementHolder]. */ context(LanguageFrontend<*, *>, Holder) -infix fun Expression.assign(rhs: Expression): BinaryOperator { - val node = (this@LanguageFrontend).newBinaryOperator("=") - node.lhs = this - node.rhs = rhs +infix fun Expression.assign(rhs: Expression): AssignExpression { + val node = (this@LanguageFrontend).newAssignExpression("=", listOf(this), listOf(rhs)) if (this@Holder is StatementHolder) { this@Holder += node @@ -1070,6 +1146,34 @@ infix fun Expression.assign(rhs: Expression): BinaryOperator { return node } +/** + * Creates a new [AssignExpression] with a `=` [AssignExpression.operatorCode] in the Fluent Node + * DSL and invokes [StatementHolder.addStatement] of the nearest enclosing [StatementHolder]. + */ +context(LanguageFrontend<*, *>, Holder) + +infix fun Expression.assignAsExpr(rhs: Expression): AssignExpression { + val node = (this@LanguageFrontend).newAssignExpression("=", listOf(this), listOf(rhs)) + + node.usedAsExpression = true + + return node +} +/** + * Creates a new [AssignExpression] with a `=` [AssignExpression.operatorCode] in the Fluent Node + * DSL and invokes [StatementHolder.addStatement] of the nearest enclosing [StatementHolder]. + */ +context(LanguageFrontend<*, *>, Holder) + +infix fun Expression.assignAsExpr(rhs: AssignExpression.() -> Unit): AssignExpression { + val node = (this@LanguageFrontend).newAssignExpression("=", listOf(this)) + rhs(node) + + node.usedAsExpression = true + + return node +} + /** Creates a new [Type] with the given [name] in the Fluent Node DSL. */ fun LanguageFrontend<*, *>.t(name: CharSequence, init: (Type.() -> Unit)? = null): Type { val type = objectType(name) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/FieldDeclaration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/FieldDeclaration.kt index b2f3c985f2..e76b68b601 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/FieldDeclaration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/FieldDeclaration.kt @@ -27,9 +27,6 @@ package de.fraunhofer.aisec.cpg.graph.declarations import de.fraunhofer.aisec.cpg.graph.* import de.fraunhofer.aisec.cpg.graph.statements.expressions.Expression -import de.fraunhofer.aisec.cpg.graph.statements.expressions.InitializerListExpression -import de.fraunhofer.aisec.cpg.graph.types.Type -import de.fraunhofer.aisec.cpg.graph.types.UnknownType import java.util.* import org.apache.commons.lang3.builder.ToStringBuilder import org.neo4j.ogm.annotation.Relationship @@ -38,26 +35,7 @@ import org.neo4j.ogm.annotation.Relationship * Declaration of a field within a [RecordDeclaration]. It contains the modifiers associated with * the field as well as an initializer [Expression] which provides an initial value for the field. */ -class FieldDeclaration : ValueDeclaration(), HasType.TypeListener, HasInitializer { - @AST - override var initializer: Expression? = null - set(value) { - if (field != null) { - isDefinition = true - field?.unregisterTypeListener(this) - if (field is HasType.TypeListener) { - unregisterTypeListener(field as HasType.TypeListener) - } - } - field = value - if (value != null) { - value.registerTypeListener(this) - if (value is HasType.TypeListener) { - registerTypeListener(value as HasType.TypeListener) - } - } - } - +class FieldDeclaration : VariableDeclaration() { /** Specifies, whether this field declaration is also a definition, i.e. has an initializer. */ private var isDefinition = false @@ -72,53 +50,8 @@ class FieldDeclaration : ValueDeclaration(), HasType.TypeListener, HasInitialize } } - /** @see VariableDeclaration.implicitInitializerAllowed */ - var isImplicitInitializerAllowed = false - - var isArray = false var modifiers: List = mutableListOf() - override fun typeChanged(src: HasType, root: MutableList, oldType: Type) { - if (!isTypeSystemActive) { - return - } - if (type !is UnknownType && src.propagationType == oldType) { - return - } - val previous = type - val newType = - if (src === initializer && initializer is InitializerListExpression) { - // Init list is seen as having an array type, but can be used ambiguously. It can be - // either used to initialize an array, or to initialize some objects. If it is used - // as an - // array initializer, we need to remove the array/pointer layer from the type, - // otherwise it - // can be ignored once we have a type - if (isArray) { - src.type - } else if (type !is UnknownType) { - return - } else { - src.type.dereference() - } - } else { - src.propagationType - } - setType(newType, root) - if (previous != type) { - type.typeOrigin = Type.Origin.DATAFLOW - } - } - - override fun possibleSubTypesChanged(src: HasType, root: MutableList) { - if (!isTypeSystemActive) { - return - } - val subTypes: MutableList = ArrayList(possibleSubTypes) - subTypes.addAll(src.possibleSubTypes) - setPossibleSubTypes(subTypes, root) - } - override fun toString(): String { return ToStringBuilder(this, TO_STRING_STYLE) .appendSuper(super.toString()) @@ -134,9 +67,7 @@ class FieldDeclaration : ValueDeclaration(), HasType.TypeListener, HasInitialize if (other !is FieldDeclaration) { return false } - return (super.equals(other) && - initializer == other.initializer && - modifiers == other.modifiers) + return (super.equals(other) && modifiers == other.modifiers) } override fun hashCode() = Objects.hash(super.hashCode(), initializer, modifiers) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/FunctionDeclaration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/FunctionDeclaration.kt index b2adfcd3c4..043737c0a4 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/FunctionDeclaration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/FunctionDeclaration.kt @@ -34,6 +34,7 @@ import de.fraunhofer.aisec.cpg.graph.statements.CompoundStatement import de.fraunhofer.aisec.cpg.graph.statements.Statement import de.fraunhofer.aisec.cpg.graph.statements.expressions.Expression import de.fraunhofer.aisec.cpg.graph.types.Type +import de.fraunhofer.aisec.cpg.graph.types.isSupertypeOf import java.util.* import java.util.stream.Collectors import org.apache.commons.lang3.builder.ToStringBuilder diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/TypeParamDeclaration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/TypeParamDeclaration.kt index a184b6d1a6..22176fd81e 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/TypeParamDeclaration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/TypeParamDeclaration.kt @@ -27,7 +27,7 @@ package de.fraunhofer.aisec.cpg.graph.declarations import de.fraunhofer.aisec.cpg.graph.AST import de.fraunhofer.aisec.cpg.graph.HasDefault -import de.fraunhofer.aisec.cpg.graph.HasType.SecondaryTypeEdge +import de.fraunhofer.aisec.cpg.graph.types.SecondaryTypeEdge import de.fraunhofer.aisec.cpg.graph.types.Type import java.util.* import org.neo4j.ogm.annotation.Relationship @@ -37,8 +37,8 @@ class TypeParamDeclaration : ValueDeclaration(), SecondaryTypeEdge, HasDefault() - /** - * The type of this declaration. In order to maximize compatibility with Java legacy code - * (primarily the type listeners), this is a virtual property which wraps around a dedicated - * backing field [_type]. - */ - override var type: Type - get() { - val result: Type = - if (isTypeSystemActive) { - _type - } else { - ctx?.typeManager - ?.typeCache - ?.computeIfAbsent(this) { mutableListOf() } - ?.firstOrNull() - ?: unknownType() - } - return result - } + /** The type of this declaration. */ + override var type: Type = unknownType() set(value) { - // Trigger the type listener foo - setType(value, null) - } + val old = field + field = value + + // Only inform our observer if the type has changed. This should not trigger if we + // "squash" types into one, because they should still be regarded as "equal", but not + // the "same". + if (old != value) { + informObservers(HasType.TypeObserver.ChangeType.TYPE) + } - @Relationship("POSSIBLE_SUB_TYPES") protected var _possibleSubTypes = mutableListOf() - override var possibleSubTypes: List - get() { - return if (!isTypeSystemActive) { - ctx?.typeManager?.typeCache?.getOrDefault(this, emptyList()) ?: listOf() - } else _possibleSubTypes + // We also want to add the definitive type (if known) to our assigned types + if (value !is UnknownType && value !is AutoType) { + addAssignedType(value) + } } + + override var assignedTypes: Set = mutableSetOf() set(value) { - setPossibleSubTypes(value, ArrayList()) - } + if (field == value) { + return + } - @Transient override val typeListeners: MutableSet = HashSet() + field = value + informObservers(HasType.TypeObserver.ChangeType.ASSIGNED_TYPE) + } /** * Links to all the [DeclaredReferenceExpression]s accessing the variable and the respective @@ -118,130 +102,6 @@ abstract class ValueDeclaration : Declaration(), HasType { usageEdges.add(usageEdge) } - /** - * There is no case in which we would want to propagate a referenceType as in this case always - * the underlying ObjectType should be propagated - * - * @return Type that should be propagated - */ - override val propagationType: Type - get() { - return if (type is ReferenceType) { - (type as ReferenceType?)?.elementType ?: unknownType() - } else type - } - - override fun setType(type: Type, root: MutableList?) { - var t: Type = type - var r: MutableList? = root - if (!isTypeSystemActive) { - cacheType(t) - return - } - if (r == null) { - r = ArrayList() - } - if ( - r.contains(this) || - t is UnknownType || - this._type is FunctionPointerType && t !is FunctionPointerType - ) { - return - } - val oldType = this.type - t = t.duplicate() - val subTypes = mutableSetOf() - for (t in possibleSubTypes) { - if (!t.isSimilar(t)) { - subTypes.add(t) - } - } - subTypes.add(t) - this._type = registerType(getCommonType(subTypes).orElse(t)) - val newSubtypes: MutableList = ArrayList() - for (s in subTypes) { - if (isSupertypeOf(this.type, s)) { - newSubtypes.add(registerType(s)) - } - } - possibleSubTypes = newSubtypes - if (oldType == t) { - // Nothing changed, so we do not have to notify the listeners. - return - } - r.add(this) // Add current node to the set of "triggers" to detect potential loops. - // Notify all listeners about the changed type - for (l in typeListeners) { - if (l != this) { - l.typeChanged(this, r, oldType) - } - } - } - - override fun setPossibleSubTypes(possibleSubTypes: List, root: MutableList) { - var list = possibleSubTypes - list = list.filterNot { type -> type is UnknownType }.distinct().toMutableList() - if (!isTypeSystemActive) { - list.forEach { t -> cacheType(t) } - - return - } - if (root.contains(this)) { - return - } - val oldSubTypes = this.possibleSubTypes - this._possibleSubTypes = list - - if (HashSet(oldSubTypes).containsAll(list)) { - // Nothing changed, so we do not have to notify the listeners. - return - } - // Add current node to the set of "triggers" to detect potential loops. - root.add(this) - - // Notify all listeners about the changed type - for (listener in typeListeners) { - if (listener != this) { - listener.possibleSubTypesChanged(this, root) - } - } - } - - override fun resetTypes(type: Type) { - val oldSubTypes = possibleSubTypes - val oldType = this._type - this._type = type - possibleSubTypes = listOf(type) - val root = mutableListOf(this) - if (oldType != type) { - typeListeners - .stream() - .filter { l: HasType.TypeListener -> l != this } - .forEach { l: HasType.TypeListener -> l.typeChanged(this, root, oldType) } - } - if (oldSubTypes.size != 1 || !oldSubTypes.contains(type)) - typeListeners - .stream() - .filter { l: HasType.TypeListener -> l != this } - .forEach { l: HasType.TypeListener -> l.possibleSubTypesChanged(this, root) } - } - - override fun refreshType() { - val root = mutableListOf(this) - for (l in typeListeners) { - l.typeChanged(this, root, type) - l.possibleSubTypesChanged(this, root) - } - } - - override fun updateType(type: Type) { - this._type = type - } - - override fun updatePossibleSubtypes(types: List) { - this._possibleSubTypes = types.toMutableList() - } - override fun toString(): String { return ToStringBuilder(this, TO_STRING_STYLE).appendSuper(super.toString()).toString() } @@ -253,9 +113,7 @@ abstract class ValueDeclaration : Declaration(), HasType { if (other !is ValueDeclaration) { return false } - return (super.equals(other) && - type == other.type && - possibleSubTypes == other.possibleSubTypes) + return (super.equals(other) && type == other.type) } override fun hashCode(): Int { diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/VariableDeclaration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/VariableDeclaration.kt index 3ca71622bb..825a10dab9 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/VariableDeclaration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/VariableDeclaration.kt @@ -26,19 +26,21 @@ package de.fraunhofer.aisec.cpg.graph.declarations import de.fraunhofer.aisec.cpg.graph.* +import de.fraunhofer.aisec.cpg.graph.statements.expressions.ConstructExpression +import de.fraunhofer.aisec.cpg.graph.statements.expressions.DeclaredReferenceExpression import de.fraunhofer.aisec.cpg.graph.statements.expressions.Expression -import de.fraunhofer.aisec.cpg.graph.statements.expressions.InitializerListExpression +import de.fraunhofer.aisec.cpg.graph.types.AutoType +import de.fraunhofer.aisec.cpg.graph.types.HasType import de.fraunhofer.aisec.cpg.graph.types.Type -import de.fraunhofer.aisec.cpg.graph.types.UnknownType import org.apache.commons.lang3.builder.ToStringBuilder import org.neo4j.ogm.annotation.Relationship -/** Represents the declaration of a variable. */ -class VariableDeclaration : ValueDeclaration(), HasType.TypeListener, HasInitializer { +/** Represents the declaration of a local variable. */ +open class VariableDeclaration : ValueDeclaration(), HasInitializer, HasType.TypeObserver { /** - * We need a way to store the templateParameters that a VariableDeclaration might have before - * the ConstructExpression is created. + * We need a way to store the templateParameters that a [VariableDeclaration] might have before + * the [ConstructExpression] is created. * * Because templates are only used by a small subset of languages and variable declarations are * used often, we intentionally make this a nullable list instead of an empty list. @@ -62,65 +64,18 @@ class VariableDeclaration : ValueDeclaration(), HasType.TypeListener, HasInitial @AST override var initializer: Expression? = null set(value) { - field?.unregisterTypeListener(this) - if (field is HasType.TypeListener) { - unregisterTypeListener(field as HasType.TypeListener) - } + field?.unregisterTypeObserver(this) field = value - value?.registerTypeListener(this) - - // if the initializer implements a type listener, inform it about our type changes - // since the type is tied to the declaration, but it is convenient to have the type - // information in the initializer, i.e. in a ConstructExpression. - if (value is HasType.TypeListener) { - registerTypeListener(value as HasType.TypeListener) + if (value is DeclaredReferenceExpression) { + value.resolutionHelper = this } + value?.registerTypeObserver(this) } fun getInitializerAs(clazz: Class): T? { return clazz.cast(initializer) } - override fun typeChanged(src: HasType, root: MutableList, oldType: Type) { - if (!isTypeSystemActive) { - return - } - if (type !is UnknownType && src.propagationType == oldType) { - return - } - val previous = type - val newType = - if (src === initializer && initializer is InitializerListExpression) { - // Init list is seen as having an array type, but can be used ambiguously. It can be - // either used to initialize an array, or to initialize some objects. If it is used - // as an - // array initializer, we need to remove the array/pointer layer from the type, - // otherwise it can be ignored once we have a type - if (isArray) { - src.type - } else if (type !is UnknownType) { - return - } else { - src.type.dereference() - } - } else { - src.propagationType - } - setType(newType, root) - if (previous != type) { - type.typeOrigin = Type.Origin.DATAFLOW - } - } - - override fun possibleSubTypesChanged(src: HasType, root: MutableList) { - if (!isTypeSystemActive) { - return - } - val subTypes: MutableList = ArrayList(possibleSubTypes) - subTypes.addAll(src.possibleSubTypes) - setPossibleSubTypes(subTypes, root) - } - override fun toString(): String { return ToStringBuilder(this, TO_STRING_STYLE) .append("name", name) @@ -129,10 +84,29 @@ class VariableDeclaration : ValueDeclaration(), HasType.TypeListener, HasInitial .toString() } - override val assignments: List - get() { - return initializer?.let { listOf(Assignment(it, this, this)) } ?: listOf() + override fun typeChanged(newType: Type, src: HasType) { + // Only accept type changes from our initializer, if any + if (src != initializer) { + return + } + + // In the auto-inference case, we want to set the type of our declaration to the + // declared type of the initializer + if (this.type is AutoType) { + type = newType + } else { + // Otherwise, we are at least interested in what the initializer's type is, to see + // whether we can fill our assigned types with that + addAssignedType(newType) + } + } + + override fun assignedTypeChanged(assignedTypes: Set, src: HasType) { + // Propagate the assigned types from our initializer into the declaration + if (src == initializer) { + addAssignedTypes(assignedTypes) } + } override fun equals(other: Any?): Boolean { if (this === other) { diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/ArrayCreationExpression.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/ArrayCreationExpression.kt index 258a9e9251..dcf37a3c17 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/ArrayCreationExpression.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/ArrayCreationExpression.kt @@ -26,13 +26,10 @@ package de.fraunhofer.aisec.cpg.graph.statements.expressions import de.fraunhofer.aisec.cpg.graph.AST -import de.fraunhofer.aisec.cpg.graph.HasType import de.fraunhofer.aisec.cpg.graph.declarations.VariableDeclaration import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge.Companion.propertyEqualsList import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdgeDelegate -import de.fraunhofer.aisec.cpg.graph.isTypeSystemActive -import de.fraunhofer.aisec.cpg.graph.types.Type import java.util.* import org.neo4j.ogm.annotation.Relationship @@ -40,7 +37,7 @@ import org.neo4j.ogm.annotation.Relationship * Expressions of the form `new Type[]` that represents the creation of an array, mostly used in * combination with a [VariableDeclaration]. */ -class ArrayCreationExpression : Expression(), HasType.TypeListener { +class ArrayCreationExpression : Expression() { /** * The initializer of the expression, if present. Many languages, such as Java, either specify * [dimensions] or an initializer. @@ -48,9 +45,7 @@ class ArrayCreationExpression : Expression(), HasType.TypeListener { @AST var initializer: Expression? = null set(value) { - field?.unregisterTypeListener(this) field = value - value?.registerTypeListener(this) } /** @@ -80,24 +75,4 @@ class ArrayCreationExpression : Expression(), HasType.TypeListener { } override fun hashCode() = Objects.hash(super.hashCode(), initializer, dimensions) - - override fun typeChanged(src: HasType, root: MutableList, oldType: Type) { - if (!isTypeSystemActive) { - return - } - val previous = type - setType(src.propagationType, root) - if (previous != type) { - type.typeOrigin = Type.Origin.DATAFLOW - } - } - - override fun possibleSubTypesChanged(src: HasType, root: MutableList) { - if (!isTypeSystemActive) { - return - } - val subTypes: MutableList = ArrayList(possibleSubTypes) - subTypes.addAll(src.possibleSubTypes) - setPossibleSubTypes(subTypes, root) - } } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/ArraySubscriptionExpression.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/ArraySubscriptionExpression.kt index 7cc3d05d6c..7206864f2c 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/ArraySubscriptionExpression.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/ArraySubscriptionExpression.kt @@ -26,16 +26,16 @@ package de.fraunhofer.aisec.cpg.graph.statements.expressions import de.fraunhofer.aisec.cpg.graph.* +import de.fraunhofer.aisec.cpg.graph.types.HasType import de.fraunhofer.aisec.cpg.graph.types.Type import java.util.* -import java.util.stream.Collectors /** * Represents the subscription or access of an array of the form `array[index]`, where both `array` * ([arrayExpression]) and `index` ([subscriptExpression]) are of type [Expression]. CPP can * overload operators thus changing semantics of array access. */ -class ArraySubscriptionExpression : Expression(), HasType.TypeListener, HasBase { +class ArraySubscriptionExpression : Expression(), HasBase, HasType.TypeObserver, ArgumentHolder { /** * The array on which the access is happening. This is most likely a * [DeclaredReferenceExpression]. @@ -43,9 +43,10 @@ class ArraySubscriptionExpression : Expression(), HasType.TypeListener, HasBase @AST var arrayExpression: Expression = ProblemExpression("could not parse array expression") set(value) { + field.unregisterTypeObserver(this) field = value type = getSubscriptType(value.type) - value.registerTypeListener(this) + value.registerTypeObserver(this) } /** @@ -75,29 +76,42 @@ class ArraySubscriptionExpression : Expression(), HasType.TypeListener, HasBase } } - override fun typeChanged(src: HasType, root: MutableList, oldType: Type) { - if (!isTypeSystemActive) { + override fun typeChanged(newType: Type, src: HasType) { + // Make sure the source is really our array + if (src != arrayExpression) { return } - val previous = type - setType(getSubscriptType(src.propagationType), root) - if (previous != type) { - type.typeOrigin = Type.Origin.DATAFLOW - } + + this.type = getSubscriptType(newType) } - override fun possibleSubTypesChanged(src: HasType, root: MutableList) { - if (!isTypeSystemActive) { + override fun assignedTypeChanged(assignedTypes: Set, src: HasType) { + // Make sure the source is really our array + if (src != arrayExpression) { return } - val subTypes: MutableList = ArrayList(possibleSubTypes) - subTypes.addAll( - src.possibleSubTypes - .stream() - .map { arrayType: Type -> getSubscriptType(arrayType) } - .collect(Collectors.toList()) - ) - setPossibleSubTypes(subTypes, root) + + addAssignedTypes(assignedTypes.map { getSubscriptType(it) }.toSet()) + } + + override fun addArgument(expression: Expression) { + if (arrayExpression is ProblemExpression) { + arrayExpression = expression + } else if (subscriptExpression is ProblemExpression) { + subscriptExpression = expression + } + } + + override fun replaceArgument(old: Expression, new: Expression): Boolean { + return if (arrayExpression == old) { + arrayExpression = new + true + } else if (subscriptExpression == old) { + subscriptExpression = new + true + } else { + false + } } override fun equals(other: Any?): Boolean { diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/AssignExpression.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/AssignExpression.kt index e240f0e67e..d48ebec724 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/AssignExpression.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/AssignExpression.kt @@ -27,6 +27,7 @@ package de.fraunhofer.aisec.cpg.graph.statements.expressions import de.fraunhofer.aisec.cpg.graph.* import de.fraunhofer.aisec.cpg.graph.declarations.VariableDeclaration +import de.fraunhofer.aisec.cpg.graph.types.HasType import de.fraunhofer.aisec.cpg.graph.types.TupleType import de.fraunhofer.aisec.cpg.graph.types.Type import org.slf4j.Logger @@ -49,34 +50,38 @@ import org.slf4j.LoggerFactory * [usedAsExpression]. When this property is set to true (it defaults to false), we model a dataflow * from the (first) rhs to the [AssignExpression] itself. */ -class AssignExpression : Expression(), AssignmentHolder, HasType.TypeListener { +class AssignExpression : + Expression(), AssignmentHolder, ArgumentHolder, HasType.TypeObserver, HasOperatorCode { - var operatorCode: String = "=" - - @AST var lhs: List = listOf() + override var operatorCode: String = "=" @AST - var rhs: List = listOf() + var lhs: List = listOf() set(value) { - // Unregister any old type listeners - field.forEach { it.unregisterTypeListener(this) } field = value - // Register this statement as a type listener for each expression - value.forEach { - it.registerTypeListener(this) - - if (it is DeclaredReferenceExpression) { - it.access = AccessValues.WRITE + if (operatorCode == "=") { + field.forEach { (it as? DeclaredReferenceExpression)?.access = AccessValues.WRITE } + } else { + field.forEach { + (it as? DeclaredReferenceExpression)?.access = AccessValues.READWRITE } } } + @AST + var rhs: List = listOf() + set(value) { + field.forEach { it.unregisterTypeObserver(this) } + field = value + value.forEach { it.registerTypeObserver(this) } + } + /** * This property specifies, that this is actually used as an expression. Not many languages * support that. In the regular case, an assignment is a simple statement and does not hold any * value itself. */ - val usedAsExpression = false + var usedAsExpression = false /** * If this node is used an expression, this property contains a reference of the [Expression] @@ -94,7 +99,7 @@ class AssignExpression : Expression(), AssignmentHolder, HasType.TypeListener { /** * We also support compound assignments in this class, but only if the appropriate compound - * operator is set and only if there is a single-value expression on both side. + * operator is set and only if there is a single-value expression on both sides. */ val isCompoundAssignment: Boolean get() { @@ -112,47 +117,6 @@ class AssignExpression : Expression(), AssignmentHolder, HasType.TypeListener { */ override var declarations = mutableListOf() - override fun typeChanged(src: HasType, root: MutableList, oldType: Type) { - if (!isTypeSystemActive) { - return - } - - val type = src.type - - // There are now two possibilities: Either, we have a tuple type, that we need to - // deconstruct, or we have a singular type - if (type is TupleType) { - val targets = findTargets(src) - if (targets.size == type.types.size) { - // Set the corresponding type on the left-side - type.types.forEachIndexed { idx, t -> lhs.getOrNull(idx)?.type = t } - } - } else { - findTargets(src).forEach { it.type = src.propagationType } - } - - // If this is used as an expression, we also set the type accordingly - if (usedAsExpression) { - expressionValue?.propagationType?.let { setType(it, root) } - } - } - - override fun possibleSubTypesChanged(src: HasType, root: MutableList) { - if (!isTypeSystemActive) { - return - } - - // Basically, we need to find out which index on the rhs this variable belongs to and set - // the corresponding subtypes on the lhs. - val idx = rhs.indexOf(src) - if (idx == -1) { - return - } - - // Set the subtypes - lhs.getOrNull(idx)?.setPossibleSubTypes(src.possibleSubTypes, root) - } - /** Finds the value (of [rhs]) that is assigned to the particular [lhs] expression. */ fun findValue(lhsExpr: HasType): Expression? { return if (lhs.size > 1) { @@ -211,4 +175,59 @@ class AssignExpression : Expression(), AssignmentHolder, HasType.TypeListener { companion object { private val log: Logger = LoggerFactory.getLogger(Node::class.java) } + + override fun typeChanged(newType: Type, src: HasType) { + // Double-check, if the src is really from the rhs + if (!rhs.contains(src)) { + return + } + + // There are now two possibilities: Either, we have a tuple type, that we need to + // deconstruct, or we have a singular type. Now, its getting tricky. We do NOT want + // to propagate the type to the declared type, but only to the "assigned" type + if (newType is TupleType) { + val targets = findTargets(src) + if (targets.size == newType.types.size) { + // Set the corresponding type on the left-side + newType.types.forEachIndexed { idx, t -> lhs.getOrNull(idx)?.addAssignedType(t) } + } + } else { + findTargets(src).forEach { it.addAssignedType(newType) } + } + + // If this is used as an expression, we also set the type accordingly + if (usedAsExpression) { + expressionValue?.type?.let { type = it } + } + } + + override fun assignedTypeChanged(assignedTypes: Set, src: HasType) { + // Double-check, if the src is really from the rhs + if (!rhs.contains(src)) { + return + } + + // Propagate any assigned types from the source to the target + findTargets(src).forEach { it.addAssignedTypes(assignedTypes) } + } + + override fun addArgument(expression: Expression) { + if (lhs.isEmpty()) { + lhs = listOf(expression) + } else { + rhs = listOf(expression) + } + } + + override fun replaceArgument(old: Expression, new: Expression): Boolean { + return if (lhs == listOf(old)) { + lhs = listOf(new) + true + } else if (rhs == listOf(old)) { + rhs = listOf(new) + true + } else { + false + } + } } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/BinaryOperator.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/BinaryOperator.kt index 7c81f0b426..dad65f97ee 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/BinaryOperator.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/BinaryOperator.kt @@ -25,20 +25,21 @@ */ package de.fraunhofer.aisec.cpg.graph.statements.expressions +import de.fraunhofer.aisec.cpg.frontends.TranslationException import de.fraunhofer.aisec.cpg.graph.* -import de.fraunhofer.aisec.cpg.graph.types.NumericType -import de.fraunhofer.aisec.cpg.graph.types.StringType +import de.fraunhofer.aisec.cpg.graph.types.HasType import de.fraunhofer.aisec.cpg.graph.types.Type -import de.fraunhofer.aisec.cpg.graph.types.UnknownType import java.util.* import org.apache.commons.lang3.builder.ToStringBuilder /** * A binary operation expression, such as "a + b". It consists of a left hand expression (lhs), a * right hand expression (rhs) and an operatorCode. + * + * Note: For assignments, i.e., using an `=` or `+=`, etc. the [AssignExpression] MUST be used. */ open class BinaryOperator : - Expression(), HasType.TypeListener, AssignmentHolder, HasBase, ArgumentHolder { + Expression(), HasBase, HasOperatorCode, ArgumentHolder, HasType.TypeObserver { /** The left-hand expression. */ @AST var lhs: Expression = ProblemExpression("could not parse lhs") @@ -56,6 +57,7 @@ open class BinaryOperator : field = value connectNewRhs(value) } + /** The operator code. */ override var operatorCode: String? = null set(value) { @@ -64,8 +66,8 @@ open class BinaryOperator : (operatorCode in (language?.compoundAssignmentOperators ?: setOf())) || (operatorCode == "=") ) { - NodeBuilder.LOGGER.warn( - "Creating a BinaryOperator with an assignment operator code is deprecated. The class AssignExpression should be used instead." + throw TranslationException( + "Creating a BinaryOperator with an assignment operator code is not allowed. The class AssignExpression should be used instead." ) } } @@ -75,35 +77,21 @@ open class BinaryOperator : } private fun connectNewLhs(lhs: Expression) { - lhs.registerTypeListener(this) - if ("=" == operatorCode) { - if (lhs is DeclaredReferenceExpression) { - // declared reference expr is the left-hand side of an assignment -> writing to the - // var - lhs.access = AccessValues.WRITE - } - if (lhs is HasType.TypeListener) { - registerTypeListener(lhs as HasType.TypeListener) - registerTypeListener(this.lhs as HasType.TypeListener) - } - } else if (operatorCode in (language?.compoundAssignmentOperators ?: setOf())) { - if (lhs is DeclaredReferenceExpression) { - // declared reference expr is the left-hand side of an assignment -> writing to the - // var - lhs.access = AccessValues.READWRITE - } - if (lhs is HasType.TypeListener) { - registerTypeListener(lhs as HasType.TypeListener) - registerTypeListener(this.lhs as HasType.TypeListener) - } + lhs.registerTypeObserver(this) + if (lhs is DeclaredReferenceExpression && "=" == operatorCode) { + // declared reference expr is the left-hand side of an assignment -> writing to the var + lhs.access = AccessValues.WRITE + } else if ( + lhs is DeclaredReferenceExpression && + operatorCode in (language?.compoundAssignmentOperators ?: setOf()) + ) { + // declared reference expr is the left-hand side of an assignment -> writing to the var + lhs.access = AccessValues.READWRITE } } private fun disconnectOldLhs() { - lhs.unregisterTypeListener(this) - if ("=" == operatorCode && lhs is HasType.TypeListener) { - unregisterTypeListener(lhs as HasType.TypeListener) - } + lhs.unregisterTypeObserver(this) } fun getRhsAs(clazz: Class): T? { @@ -111,65 +99,11 @@ open class BinaryOperator : } private fun connectNewRhs(rhs: Expression) { - rhs.registerTypeListener(this) - if ("=" == operatorCode && rhs is HasType.TypeListener) { - registerTypeListener(rhs as HasType.TypeListener) - } + rhs.registerTypeObserver(this) } private fun disconnectOldRhs() { - rhs.unregisterTypeListener(this) - if ("=" == operatorCode && rhs is HasType.TypeListener) { - unregisterTypeListener(rhs as HasType.TypeListener) - } - } - - override fun typeChanged(src: HasType, root: MutableList, oldType: Type) { - if (!isTypeSystemActive) { - return - } - val previous = type - if (operatorCode == "=") { - val srcWidth = (src.type as? NumericType)?.bitWidth - val lhsWidth = (lhs.type as? NumericType)?.bitWidth - if (src == rhs && lhsWidth != null && srcWidth != null && lhsWidth < srcWidth) { - // Do not propagate anything if the new type is too big for the current type. - return - } - setType(src.propagationType, root) - } else if ( - operatorCode == "+" && - (lhs.propagationType is StringType || rhs.propagationType is StringType) - ) { - // String + any other type results in a String - _possibleSubTypes.clear() // TODO: Why do we clear the list here? - val stringType = - if (lhs.propagationType is StringType) lhs.propagationType else rhs.propagationType - setType(stringType, root) - } else if (operatorCode == ".*" || operatorCode == "->*" && src === rhs) { - // Propagate the function pointer type to the expression itself. This helps us later in - // the call resolver, when trying to determine, whether this is a regular call or a - // function pointer call. - setType(src.propagationType, root) - } else { - val resultingType = language?.propagateTypeOfBinaryOperation(this) ?: unknownType() - if (resultingType !is UnknownType) { - setType(resultingType, root) - } - } - - if (previous != type) { - type.typeOrigin = Type.Origin.DATAFLOW - } - } - - override fun possibleSubTypesChanged(src: HasType, root: MutableList) { - if (!isTypeSystemActive) { - return - } - val subTypes: MutableList = ArrayList(possibleSubTypes) - subTypes.addAll(src.possibleSubTypes) - setPossibleSubTypes(subTypes, root) + rhs.unregisterTypeObserver(this) } override fun toString(): String { @@ -180,15 +114,29 @@ open class BinaryOperator : .toString() } - @Deprecated("BinaryOperator should not be used for assignments anymore") - override val assignments: List - get() { - return if (isAssignment) { - listOf(Assignment(rhs, lhs, this)) + override fun typeChanged(newType: Type, src: HasType) { + // We need to do some special dealings for function pointer calls + if (operatorCode == ".*" || operatorCode == "->*" && src === rhs) { + // Propagate the function pointer type to the expression itself. This helps us later in + // the call resolver, when trying to determine, whether this is a regular call or a + // function pointer call. + this.type = newType + } else { + // Otherwise, we have a special language-specific function to deal with type propagation + val type = language?.propagateTypeOfBinaryOperation(this) + if (type != null) { + this.type = type } else { - listOf() + // If we don't know how to propagate the types of this particular binary operation, + // we just leave the type alone. We cannot take newType because it is just "half" of + // the operation (either from lhs or rhs) and would lead to very incorrect results. } } + } + + override fun assignedTypeChanged(assignedTypes: Set, src: HasType) { + // TODO: replicate something similar like propagateTypeOfBinaryOperation for assigned types + } override fun equals(other: Any?): Boolean { if (this === other) { @@ -205,15 +153,6 @@ open class BinaryOperator : override fun hashCode() = Objects.hash(super.hashCode(), lhs, rhs, operatorCode) - private val isAssignment: Boolean - get() { - // TODO(oxisto): We need to discuss, if the other operators are also assignments and if - // we really want them - return this.operatorCode.equals("=") - /*||this.operatorCode.equals("+=") ||this.operatorCode.equals("-=") - ||this.operatorCode.equals("/=") ||this.operatorCode.equals("*=")*/ - } - override fun addArgument(expression: Expression) { if (lhs is ProblemExpression) { lhs = expression diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/CallExpression.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/CallExpression.kt index 83d77fab03..93d0e0fc5a 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/CallExpression.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/CallExpression.kt @@ -27,7 +27,6 @@ package de.fraunhofer.aisec.cpg.graph.statements.expressions import de.fraunhofer.aisec.cpg.PopulatedByPass import de.fraunhofer.aisec.cpg.graph.* -import de.fraunhofer.aisec.cpg.graph.HasType.SecondaryTypeEdge import de.fraunhofer.aisec.cpg.graph.declarations.FunctionDeclaration import de.fraunhofer.aisec.cpg.graph.declarations.TemplateDeclaration import de.fraunhofer.aisec.cpg.graph.declarations.TemplateDeclaration.TemplateInitialization @@ -36,9 +35,8 @@ import de.fraunhofer.aisec.cpg.graph.edge.Properties import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge.Companion.propertyEqualsList import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge.Companion.transformIntoOutgoingPropertyEdgeList import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge.Companion.unwrap -import de.fraunhofer.aisec.cpg.graph.types.FunctionPointerType -import de.fraunhofer.aisec.cpg.graph.types.TupleType -import de.fraunhofer.aisec.cpg.graph.types.Type +import de.fraunhofer.aisec.cpg.graph.types.* +import de.fraunhofer.aisec.cpg.graph.types.SecondaryTypeEdge import de.fraunhofer.aisec.cpg.passes.CallResolver import de.fraunhofer.aisec.cpg.passes.VariableUsageResolver import java.util.* @@ -49,8 +47,11 @@ import org.neo4j.ogm.annotation.Relationship * An expression, which calls another function. It has a list of arguments (list of [Expression]s) * and is connected via the INVOKES edge to its [FunctionDeclaration]. */ -open class CallExpression : Expression(), HasType.TypeListener, SecondaryTypeEdge, ArgumentHolder { - /** Connection to its [FunctionDeclaration]. This will be populated by the [CallResolver]. */ +open class CallExpression : Expression(), HasType.TypeObserver, SecondaryTypeEdge, ArgumentHolder { + /** + * Connection to its [FunctionDeclaration]. This will be populated by the [CallResolver]. This + * will have an effect on the [type] + */ @PopulatedByPass(CallResolver::class) @Relationship(value = "INVOKES", direction = Relationship.Direction.OUTGOING) var invokeEdges = mutableListOf>() @@ -70,9 +71,9 @@ open class CallExpression : Expression(), HasType.TypeListener, SecondaryTypeEdg return Collections.unmodifiableList(targets) } set(value) { - unwrap(invokeEdges).forEach { it.unregisterTypeListener(this) } + unwrap(invokeEdges).forEach { it.unregisterTypeObserver(this) } invokeEdges = transformIntoOutgoingPropertyEdgeList(value, this) - value.forEach { it.registerTypeListener(this) } + value.forEach { it.registerTypeObserver(this) } } /** @@ -265,18 +266,14 @@ open class CallExpression : Expression(), HasType.TypeListener, SecondaryTypeEdg return templateInstantiation != null || templateParameterEdges != null || template } - override fun typeChanged(src: HasType, root: MutableList, oldType: Type) { - if (!isTypeSystemActive) { - return - } - + override fun typeChanged(newType: Type, src: HasType) { // If this is a template, we need to ignore incoming type changes, because our template // system will explicitly set the type if (this.template) { return } - val previous = type + // TODO(oxisto): We could actually use the newType (which is a FunctionType now) val types = invokeEdges.map(PropertyEdge::end).mapNotNull { if (it.returnTypes.size == 1) { @@ -288,25 +285,12 @@ open class CallExpression : Expression(), HasType.TypeListener, SecondaryTypeEdg } val alternative = if (types.isNotEmpty()) types[0] else unknownType() val commonType = getCommonType(types).orElse(alternative) - val subTypes: MutableList = ArrayList(possibleSubTypes) - - subTypes.remove(oldType) - subTypes.addAll(types) - setType(commonType, root) - setPossibleSubTypes(subTypes, root) - if (previous != type) { - type.typeOrigin = Type.Origin.DATAFLOW - } - } - override fun possibleSubTypesChanged(src: HasType, root: MutableList) { - if (!isTypeSystemActive) { - return - } + this.type = commonType + } - val subTypes: MutableList = ArrayList(possibleSubTypes) - subTypes.addAll(src.possibleSubTypes) - setPossibleSubTypes(subTypes, root) + override fun assignedTypeChanged(assignedTypes: Set, src: HasType) { + // Nothing to do } override fun toString(): String { diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/CastExpression.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/CastExpression.kt index e2073c202d..ca8b592b8f 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/CastExpression.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/CastExpression.kt @@ -26,13 +26,19 @@ package de.fraunhofer.aisec.cpg.graph.statements.expressions import de.fraunhofer.aisec.cpg.graph.* +import de.fraunhofer.aisec.cpg.graph.types.HasType import de.fraunhofer.aisec.cpg.graph.types.Type import java.util.* -import kotlin.collections.ArrayList import org.slf4j.LoggerFactory -class CastExpression : Expression(), HasType.TypeListener { - @AST var expression: Expression = ProblemExpression("could not parse inner expression") +class CastExpression : Expression(), ArgumentHolder, HasType.TypeObserver { + @AST + var expression: Expression = ProblemExpression("could not parse inner expression") + set(value) { + field.unregisterTypeObserver(this) + field = value + value.registerTypeObserver(this) + } var castType: Type = unknownType() set(value) { @@ -40,33 +46,6 @@ class CastExpression : Expression(), HasType.TypeListener { type = value } - override fun updateType(type: Type) { - super.updateType(type) - castType = type - } - - override fun typeChanged(src: HasType, root: MutableList, oldType: Type) { - if (!isTypeSystemActive) { - return - } - val previous = type - if (isSupertypeOf(castType, src.propagationType)) { - setType(src.propagationType, root) - } else { - resetTypes(castType) - } - if (previous != type) { - type.typeOrigin = Type.Origin.DATAFLOW - } - } - - override fun possibleSubTypesChanged(src: HasType, root: MutableList) { - if (!isTypeSystemActive) { - return - } - setPossibleSubTypes(ArrayList(src.possibleSubTypes), root) - } - fun setCastOperator(operatorCode: Int) { var localName: String? = null when (operatorCode) { @@ -82,6 +61,30 @@ class CastExpression : Expression(), HasType.TypeListener { } } + override fun addArgument(expression: Expression) { + this.expression = expression + } + + override fun replaceArgument(old: Expression, new: Expression): Boolean { + if (this.expression == old) { + this.expression = new + return true + } + + return false + } + + override fun typeChanged(newType: Type, src: HasType) { + // Nothing to do, the cast type always stays the same + } + + override fun assignedTypeChanged(assignedTypes: Set, src: HasType) { + // We want to propagate the assigned types, if they come from our expression + if (src == expression) { + addAssignedTypes(assignedTypes) + } + } + override fun equals(other: Any?): Boolean { if (this === other) { return true diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/ConditionalExpression.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/ConditionalExpression.kt index 6fc9049a9f..0374329164 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/ConditionalExpression.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/ConditionalExpression.kt @@ -26,8 +26,9 @@ package de.fraunhofer.aisec.cpg.graph.statements.expressions import de.fraunhofer.aisec.cpg.graph.* +import de.fraunhofer.aisec.cpg.graph.types.HasType import de.fraunhofer.aisec.cpg.graph.types.Type -import java.util.ArrayList +import de.fraunhofer.aisec.cpg.graph.types.getCommonType import java.util.Objects import org.apache.commons.lang3.builder.ToStringBuilder @@ -35,55 +36,25 @@ import org.apache.commons.lang3.builder.ToStringBuilder * Represents an expression containing a ternary operator: `var x = condition ? valueIfTrue : * valueIfFalse`; */ -class ConditionalExpression : Expression(), HasType.TypeListener, ArgumentHolder, BranchingNode { +class ConditionalExpression : Expression(), ArgumentHolder, BranchingNode, HasType.TypeObserver { @AST var condition: Expression = ProblemExpression("could not parse condition expression") @AST var thenExpr: Expression? = null set(value) { - field?.unregisterTypeListener(this) + field?.unregisterTypeObserver(this) field = value - value?.registerTypeListener(this) + value?.registerTypeObserver(this) } @AST var elseExpr: Expression? = null set(value) { - field?.unregisterTypeListener(this) + field?.unregisterTypeObserver(this) field = value - value?.registerTypeListener(this) + value?.registerTypeObserver(this) } - override fun typeChanged(src: HasType, root: MutableList, oldType: Type) { - if (!isTypeSystemActive) { - return - } - val previous = type - val types: MutableList = ArrayList() - - thenExpr?.propagationType?.let { types.add(it) } - elseExpr?.propagationType?.let { types.add(it) } - - val subTypes: MutableList = ArrayList(possibleSubTypes) - subTypes.remove(oldType) - subTypes.addAll(types) - val alternative = if (types.isNotEmpty()) types[0] else unknownType() - setType(getCommonType(types).orElse(alternative), root) - setPossibleSubTypes(subTypes, root) - if (previous != type) { - type.typeOrigin = Type.Origin.DATAFLOW - } - } - - override fun possibleSubTypesChanged(src: HasType, root: MutableList) { - if (!isTypeSystemActive) { - return - } - val subTypes: MutableList = ArrayList(possibleSubTypes) - subTypes.addAll(src.possibleSubTypes) - possibleSubTypes = subTypes - } - override fun toString(): String { return ToStringBuilder(this, TO_STRING_STYLE) .appendSuper(super.toString()) @@ -105,6 +76,26 @@ class ConditionalExpression : Expression(), HasType.TypeListener, ArgumentHolder return false } + override fun typeChanged(newType: Type, src: HasType) { + val types = mutableListOf() + + thenExpr?.type?.let { types.add(it) } + elseExpr?.type?.let { types.add(it) } + + val alternative = if (types.isNotEmpty()) types[0] else unknownType() + this.type = getCommonType(types).orElse(alternative) + } + + override fun assignedTypeChanged(assignedTypes: Set, src: HasType) { + // Merge and propagate the assigned types of our branches + if (src == thenExpr || src == elseExpr) { + val types = mutableSetOf() + thenExpr?.assignedTypes?.let { types.addAll(it) } + elseExpr?.assignedTypes?.let { types.addAll(it) } + addAssignedTypes(types) + } + } + override fun equals(other: Any?): Boolean { if (this === other) return true if (other !is ConditionalExpression) return false diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/ConstructExpression.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/ConstructExpression.kt index 47c9884711..6c3e413591 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/ConstructExpression.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/ConstructExpression.kt @@ -28,8 +28,6 @@ package de.fraunhofer.aisec.cpg.graph.statements.expressions import de.fraunhofer.aisec.cpg.PopulatedByPass import de.fraunhofer.aisec.cpg.graph.* import de.fraunhofer.aisec.cpg.graph.declarations.* -import de.fraunhofer.aisec.cpg.graph.types.FunctionType -import de.fraunhofer.aisec.cpg.graph.types.Type import de.fraunhofer.aisec.cpg.graph.types.UnknownType import de.fraunhofer.aisec.cpg.passes.CallResolver import java.util.* @@ -41,7 +39,7 @@ import org.apache.commons.lang3.builder.ToStringBuilder * as part of a [NewExpression]. * * In Java, it is the initializer of a [NewExpression]. */ -class ConstructExpression : CallExpression(), HasType.TypeListener { +class ConstructExpression : CallExpression() { /** * The link to the [ConstructorDeclaration]. This is populated by the * [de.fraunhofer.aisec.cpg.passes.CallResolver] later. @@ -81,46 +79,6 @@ class ConstructExpression : CallExpression(), HasType.TypeListener { } } - /** - * This function implements the [HasType.TypeListener] interface. We need to be really careful - * about type changes in the [ConstructExpression]. The problem is, that usually, a - * [VariableDeclaration] is registered as a type listener for its initializer, e.g, to infer the - * type of the variable declaration based on its literal initializer. BUT, if the initializer - * also implements [HasType.TypeListener], as does [ConstructExpression], the initializer is - * also registered as a type listener for the declaration. The reason for that is primary - * stemming from the way the C++ AST works where we need to get information about `Integer - * i(4)`, in which the `Integer` type is only available to the declaration AST element and `(4)` - * which is the [ConstructExpression] does not have the type information. - * - * Furthermore, there is a second source of type listening events coming from the [CallResolver] - * , more specifically, if [CallExpression.invokes] is set. In this case, the call target, i.e., - * the [ConstructorDeclaration] invokes this function here. We have to differentiate between - * those two, because in the second case we are not interested in the full - * [FunctionDeclaration.type] that propagates this change (which is a [FunctionType], but only - * its [FunctionDeclaration.returnTypes]. This is already handled by - * [CallExpression.typeChanged], so we can just delegate to that. - * - * In fact, we could get rid of this particular implementation altogether, if we would somehow - * work around the first case in a different way. - */ - override fun typeChanged(src: HasType, root: MutableList, oldType: Type) { - if (!isTypeSystemActive) { - return - } - - // In the second case (see above), the src is always a function declaration, so we can - // delegate this to our parent. - if (src is FunctionDeclaration) { - return super.typeChanged(src, root, oldType) - } - - val previous: Type = this.type - setType(src.propagationType, root) - if (previous != this.type) { - this.type.typeOrigin = Type.Origin.DATAFLOW - } - } - override fun toString(): String { return ToStringBuilder(this, TO_STRING_STYLE) .appendSuper(super.toString()) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/DeclaredReferenceExpression.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/DeclaredReferenceExpression.kt index 10859e88de..0a7b589939 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/DeclaredReferenceExpression.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/DeclaredReferenceExpression.kt @@ -27,39 +27,38 @@ package de.fraunhofer.aisec.cpg.graph.statements.expressions import de.fraunhofer.aisec.cpg.PopulatedByPass import de.fraunhofer.aisec.cpg.graph.AccessValues -import de.fraunhofer.aisec.cpg.graph.HasType +import de.fraunhofer.aisec.cpg.graph.Node import de.fraunhofer.aisec.cpg.graph.declarations.Declaration import de.fraunhofer.aisec.cpg.graph.declarations.ValueDeclaration import de.fraunhofer.aisec.cpg.graph.declarations.VariableDeclaration -import de.fraunhofer.aisec.cpg.graph.isTypeSystemActive +import de.fraunhofer.aisec.cpg.graph.edge.Properties +import de.fraunhofer.aisec.cpg.graph.types.HasType import de.fraunhofer.aisec.cpg.graph.types.Type import de.fraunhofer.aisec.cpg.passes.VariableUsageResolver import java.util.* -import kotlin.collections.ArrayList import org.apache.commons.lang3.builder.ToStringBuilder import org.neo4j.ogm.annotation.Relationship /** * An expression, which refers to something which is declared, e.g. a variable. For example, the - * expression `a = b`, which itself is a [BinaryOperator], contains two [ ]s, one for the variable - * `a` and one for variable `b ` * , which have been previously been declared. + * expression `a = b`, which itself is an [AssignExpression], contains two + * [DeclaredReferenceExpression]s, one for the variable `a` and one for variable `b`, which have + * been previously been declared. */ -open class DeclaredReferenceExpression : Expression(), HasType.TypeListener { - /** The [Declaration]s this expression might refer to. */ +open class DeclaredReferenceExpression : Expression(), HasType.TypeObserver { + /** + * The [Declaration]s this expression might refer to. This will influence the [declaredType] of + * this expression. + */ @PopulatedByPass(VariableUsageResolver::class) @Relationship(value = "REFERS_TO") var refersTo: Declaration? = null set(value) { val current = field - // unregister type listeners for current declaration - if (current != null) { - if (current is ValueDeclaration) { - current.unregisterTypeListener(this) - } - if (current is HasType.TypeListener) { - unregisterTypeListener(current as HasType.TypeListener) - } + // unregister type observers for current declaration + if (current != null && current is HasType) { + current.unregisterTypeObserver(this) } // set it @@ -68,12 +67,9 @@ open class DeclaredReferenceExpression : Expression(), HasType.TypeListener { value.addUsage(this) } - // update type listeners - if (field is ValueDeclaration) { - (field as ValueDeclaration).registerTypeListener(this) - } - if (field is HasType.TypeListener) { - registerTypeListener(field as HasType.TypeListener) + // Register ourselves to get type updates from the declaration + if (value is HasType) { + value.registerTypeObserver(this) } } // set the access @@ -85,7 +81,14 @@ open class DeclaredReferenceExpression : Expression(), HasType.TypeListener { var isStaticAccess = false /** - * Returns the contents of [.refersTo] as the specified class, if the class is assignable. + * This is a MAJOR workaround needed to resolve function pointers, until we properly re-design + * the call resolver. When this [DeclaredReferenceExpression] contains a function pointer + * reference that is assigned to a variable (or to another reference), we need to set + */ + var resolutionHelper: HasType? = null + + /** + * Returns the contents of [refersTo] as the specified class, if the class is assignable. * Otherwise, it will return null. * * @param clazz the expected class @@ -99,30 +102,6 @@ open class DeclaredReferenceExpression : Expression(), HasType.TypeListener { else null } - override fun typeChanged(src: HasType, root: MutableList, oldType: Type) { - if (!isTypeSystemActive) { - return - } - val previous = type - setType(src.propagationType, root) - if (previous != type) { - type.typeOrigin = Type.Origin.DATAFLOW - } - } - - override fun possibleSubTypesChanged(src: HasType, root: MutableList) { - if (!isTypeSystemActive) { - return - } - - // since we want to update the sub types, we need to exclude ourselves from the root, - // otherwise it won't work. What a weird and broken system! - root.remove(this) - val subTypes: MutableList = ArrayList(possibleSubTypes) - subTypes.addAll(src.possibleSubTypes) - setPossibleSubTypes(subTypes, root) - } - override fun toString(): String { return ToStringBuilder(this, TO_STRING_STYLE) .append(super.toString()) @@ -130,6 +109,27 @@ open class DeclaredReferenceExpression : Expression(), HasType.TypeListener { .toString() } + override fun typeChanged(newType: Type, src: HasType) { + // Make sure that the update comes from our declaration, if we change our declared type + if (src == refersTo) { + // Set our type + this.type = newType + } + } + + override fun assignedTypeChanged(assignedTypes: Set, src: HasType) { + // Make sure that the update comes from our declaration, if we change our assigned types + if (src == refersTo) { + // Set our type + this.addAssignedTypes(assignedTypes) + } + + // We also allow updates from our previous DFG nodes + if (prevDFG.contains(src as Node)) { + this.addAssignedTypes(assignedTypes) + } + } + override fun equals(other: Any?): Boolean { if (this === other) { return true @@ -140,5 +140,16 @@ open class DeclaredReferenceExpression : Expression(), HasType.TypeListener { return super.equals(other) && refersTo == other.refersTo } + override fun addPrevDFG(prev: Node, properties: MutableMap) { + super.addPrevDFG(prev, properties) + + // We want to propagate assigned types all through the previous DFG nodes. Therefore, we + // override the DFG adding function here and add a type observer to the previous node (if it + // is not ourselves) + if (prev != this && prev is HasType) { + prev.registerTypeObserver(this) + } + } + override fun hashCode(): Int = Objects.hash(super.hashCode(), refersTo) } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/Expression.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/Expression.kt index a690bf68fc..db1e6563ab 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/Expression.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/Expression.kt @@ -27,13 +27,8 @@ package de.fraunhofer.aisec.cpg.graph.statements.expressions import de.fraunhofer.aisec.cpg.graph.* import de.fraunhofer.aisec.cpg.graph.statements.Statement -import de.fraunhofer.aisec.cpg.graph.types.FunctionPointerType -import de.fraunhofer.aisec.cpg.graph.types.ReferenceType -import de.fraunhofer.aisec.cpg.graph.types.Type -import de.fraunhofer.aisec.cpg.graph.types.UnknownType -import java.util.* +import de.fraunhofer.aisec.cpg.graph.types.* import org.apache.commons.lang3.builder.ToStringBuilder -import org.neo4j.ogm.annotation.Relationship import org.neo4j.ogm.annotation.Transient /** @@ -48,185 +43,35 @@ import org.neo4j.ogm.annotation.Transient *

This is not possible in Java, the aforementioned code example would prompt a compile error. */ abstract class Expression : Statement(), HasType { + @Transient override val typeObservers = mutableListOf() - @Relationship("TYPE") private var _type: Type = unknownType() - - /** The type of the value after evaluation. */ - override var type: Type - get() { - val result: Type = - if (isTypeSystemActive) { - _type - } else { - ctx?.typeManager - ?.typeCache - ?.computeIfAbsent(this) { mutableListOf() } - ?.firstOrNull() - ?: unknownType() - } - return result - } - set(value) { - // Trigger the type listener foo - setType(value, null) - } - - @Relationship("POSSIBLE_SUB_TYPES") protected var _possibleSubTypes = mutableListOf() - - override var possibleSubTypes: List - get() { - return if (!isTypeSystemActive) { - ctx?.typeManager?.typeCache?.getOrDefault(this, emptyList()) ?: listOf() - } else _possibleSubTypes - } + override var type: Type = unknownType() set(value) { - setPossibleSubTypes(value, ArrayList()) - } - - @Transient override val typeListeners: MutableSet = HashSet() - - override val propagationType: Type - get() { - return if (type is ReferenceType) { - (type as ReferenceType?)?.elementType ?: unknownType() - } else type - } - - @Override - override fun setType(type: Type, root: MutableList?) { - var t: Type = type - var r: MutableList? = root - - // TODO Document this method. It is called very often (potentially for each AST node) and - // performs less than optimal. - if (!isTypeSystemActive) { - this._type = t - cacheType(t) - return - } - - if (r == null) { - r = mutableListOf() - } - - // No (or only unknown) type given, loop detected? Stop early because there's nothing we can - // do. - if ( - r.contains(this) || - t is UnknownType || - stopPropagation(this.type, t) || - (this.type is FunctionPointerType && t !is FunctionPointerType) - ) { - return - } - - val oldType = this.type - // Backup to check if something changed - - t = t.duplicate() - val subTypes = mutableSetOf() - - // Check all current subtypes and consider only those which are "different enough" to type. - for (s in possibleSubTypes) { - if (!s.isSimilar(s)) { - subTypes.add(s) - } - } - - subTypes.add(t) - - // Probably tries to get something like the best supertype of all possible subtypes. - this._type = registerType(getCommonType(subTypes).orElse(t)) - - // TODO: Why do we need this loop? Shouldn't the condition be ensured by the previous line - // getting the common type?? - val newSubtypes = mutableListOf() - for (s in subTypes) { - if (isSupertypeOf(this.type, s)) { - newSubtypes.add(registerType(s)) + val old = field + field = value + + // Only inform our observer if the type has changed. This should not trigger if we + // "squash" types into one, because they should still be regarded as "equal", but not + // the "same". + if (old != value) { + informObservers(HasType.TypeObserver.ChangeType.TYPE) } - } - - possibleSubTypes = newSubtypes - - if (oldType == t) { - // Nothing changed, so we do not have to notify the listeners. - return - } - // Add current node to the set of "triggers" to detect potential loops. - r.add(this) - - // Notify all listeners about the changed type - for (l in typeListeners) { - if (l != this) { - l.typeChanged(this, r, oldType) + // We also want to add the definitive type (if known) to our assigned types + if (value !is UnknownType && value !is AutoType) { + addAssignedType(value) } } - } - - override fun setPossibleSubTypes(possibleSubTypes: List, root: MutableList) { - var list = possibleSubTypes - list = list.filterNot { type -> type is UnknownType }.distinct().toMutableList() - if (!isTypeSystemActive) { - list.forEach { t -> cacheType(t) } - return - } - if (root.contains(this)) { - return - } - val oldSubTypes = this.possibleSubTypes - this._possibleSubTypes = list - - if (HashSet(oldSubTypes).containsAll(list)) { - // Nothing changed, so we do not have to notify the listeners. - return - } - // Add current node to the set of "triggers" to detect potential loops. - root.add(this) - // Notify all listeners about the changed type - for (listener in typeListeners) { - if (listener != this) { - listener.possibleSubTypesChanged(this, root) + override var assignedTypes: Set = mutableSetOf() + set(value) { + if (field == value) { + return } - } - } - - override fun resetTypes(type: Type) { - val oldSubTypes = possibleSubTypes - val oldType = this._type - this._type = type - possibleSubTypes = listOf(type) - val root = mutableListOf(this) - if (oldType != type) { - typeListeners - .stream() - .filter { l: HasType.TypeListener -> l != this } - .forEach { l: HasType.TypeListener -> l.typeChanged(this, root, oldType) } - } - if (oldSubTypes.size != 1 || !oldSubTypes.contains(type)) - typeListeners - .stream() - .filter { l: HasType.TypeListener -> l != this } - .forEach { l: HasType.TypeListener -> l.possibleSubTypesChanged(this, root) } - } - override fun refreshType() { - val root = mutableListOf(this) - for (l in typeListeners) { - l.typeChanged(this, root, type) - l.possibleSubTypesChanged(this, root) + field = value + informObservers(HasType.TypeObserver.ChangeType.ASSIGNED_TYPE) } - } - - override fun updateType(type: Type) { - this._type = type - } - - override fun updatePossibleSubtypes(types: List) { - this._possibleSubTypes = types.toMutableList() - } override fun toString(): String { return ToStringBuilder(this, TO_STRING_STYLE) @@ -242,9 +87,7 @@ abstract class Expression : Statement(), HasType { if (other !is Expression) { return false } - return (super.equals(other) && - type == other.type && - possibleSubTypes == other.possibleSubTypes) + return (super.equals(other) && type == other.type) } override fun hashCode(): Int { diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/ExpressionList.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/ExpressionList.kt index a82869e10d..28de83ce55 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/ExpressionList.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/ExpressionList.kt @@ -26,72 +26,26 @@ package de.fraunhofer.aisec.cpg.graph.statements.expressions import de.fraunhofer.aisec.cpg.graph.AST -import de.fraunhofer.aisec.cpg.graph.HasType import de.fraunhofer.aisec.cpg.graph.edge.Properties import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge.Companion.propertyEqualsList -import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge.Companion.transformIntoOutgoingPropertyEdgeList -import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge.Companion.unwrap -import de.fraunhofer.aisec.cpg.graph.isTypeSystemActive +import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdgeDelegate import de.fraunhofer.aisec.cpg.graph.statements.Statement -import de.fraunhofer.aisec.cpg.graph.types.Type import java.util.* import kotlin.collections.ArrayList import org.neo4j.ogm.annotation.Relationship -class ExpressionList : Expression(), HasType.TypeListener { +class ExpressionList : Expression() { @Relationship(value = "SUBEXPR", direction = Relationship.Direction.OUTGOING) @AST var expressionEdges: MutableList> = ArrayList() - var expressions: List - get() { - return unwrap(expressionEdges) - } - set(value) { - if (this.expressionEdges.isNotEmpty()) { - val lastExpression = this.expressionEdges[this.expressionEdges.size - 1].end - if (lastExpression is HasType) - (lastExpression as HasType).unregisterTypeListener(this) - } - this.expressionEdges = transformIntoOutgoingPropertyEdgeList(value, this) - if (this.expressionEdges.isNotEmpty()) { - val lastExpression = this.expressionEdges[this.expressionEdges.size - 1].end - if (lastExpression is HasType) - (lastExpression as HasType).registerTypeListener(this) - } - } + var expressions: List by PropertyEdgeDelegate(ExpressionList::expressionEdges, true) fun addExpression(expression: Statement) { - if (expressionEdges.isNotEmpty()) { - val lastExpression = expressionEdges[expressionEdges.size - 1].end - if (lastExpression is HasType) (lastExpression as HasType).unregisterTypeListener(this) - } val propertyEdge = PropertyEdge(this, expression) propertyEdge.addProperty(Properties.INDEX, expressionEdges.size) expressionEdges.add(propertyEdge) - if (expression is HasType) { - (expression as HasType).registerTypeListener(this) - } - } - - override fun typeChanged(src: HasType, root: MutableList, oldType: Type) { - if (!isTypeSystemActive) { - return - } - val previous = type - setType(src.propagationType, root) - setPossibleSubTypes(ArrayList(src.possibleSubTypes), root) - if (previous != type) { - type.typeOrigin = Type.Origin.DATAFLOW - } - } - - override fun possibleSubTypesChanged(src: HasType, root: MutableList) { - if (!isTypeSystemActive) { - return - } - setPossibleSubTypes(ArrayList(src.possibleSubTypes), root) } override fun equals(other: Any?): Boolean { diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/InitializerListExpression.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/InitializerListExpression.kt index 69f7858c1c..c937cc65bb 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/InitializerListExpression.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/InitializerListExpression.kt @@ -29,74 +29,33 @@ import de.fraunhofer.aisec.cpg.graph.* import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge.Companion.propertyEqualsList import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdgeDelegate -import de.fraunhofer.aisec.cpg.graph.types.PointerType.PointerOrigin +import de.fraunhofer.aisec.cpg.graph.types.HasType +import de.fraunhofer.aisec.cpg.graph.types.PointerType import de.fraunhofer.aisec.cpg.graph.types.Type -import de.fraunhofer.aisec.cpg.graph.types.UnknownType import java.util.* import org.apache.commons.lang3.builder.ToStringBuilder import org.neo4j.ogm.annotation.Relationship -/** A list of initializer expressions. */ -class InitializerListExpression : Expression(), HasType.TypeListener { +/** + * This node represents the initialization of an "aggregate" object, such as an array or a struct or + * object. The actual use can greatly differ by the individual language frontends. In order to be as + * accurate as possible when propagating types, the [InitializerListExpression.type] property MUST + * be set before adding any values to [InitializerListExpression.initializers]. + */ +class InitializerListExpression : Expression(), ArgumentHolder, HasType.TypeObserver { /** The list of initializers. */ @Relationship(value = "INITIALIZERS", direction = Relationship.Direction.OUTGOING) @AST var initializerEdges = mutableListOf>() set(value) { - field.forEach { - it.end.unregisterTypeListener(this) - removePrevDFG(it.end) - } + field.forEach { it.end.unregisterTypeObserver(this) } field = value - value.forEach { - it.end.registerTypeListener(this) - addPrevDFG(it.end) - } + value.forEach { it.end.registerTypeObserver(this) } } /** Virtual property to access [initializerEdges] without property edges. */ var initializers by PropertyEdgeDelegate(InitializerListExpression::initializerEdges) - override fun typeChanged(src: HasType, root: MutableList, oldType: Type) { - if (!isTypeSystemActive) { - return - } - if (type !is UnknownType && src.propagationType == oldType) { - return - } - val previous = type - val newType: Type - val subTypes: MutableList - if (initializers.contains(src)) { - val types = - initializers.map { registerType(it.type.reference(PointerOrigin.ARRAY)) }.toSet() - val alternative = if (types.isNotEmpty()) types.iterator().next() else unknownType() - newType = getCommonType(types).orElse(alternative) - subTypes = ArrayList(possibleSubTypes) - subTypes.remove(oldType) - subTypes.addAll(types) - } else { - newType = src.type - subTypes = ArrayList(possibleSubTypes) - subTypes.remove(oldType) - subTypes.add(newType) - } - setType(newType, root) - setPossibleSubTypes(subTypes, root) - if (previous != type) { - type.typeOrigin = Type.Origin.DATAFLOW - } - } - - override fun possibleSubTypesChanged(src: HasType, root: MutableList) { - if (!isTypeSystemActive) { - return - } - val subTypes: MutableList = ArrayList(possibleSubTypes) - subTypes.addAll(src.possibleSubTypes) - setPossibleSubTypes(subTypes, root) - } - override fun toString(): String { return ToStringBuilder(this, TO_STRING_STYLE) .appendSuper(super.toString()) @@ -104,6 +63,40 @@ class InitializerListExpression : Expression(), HasType.TypeListener { .toString() } + override fun addArgument(expression: Expression) { + this.initializers += expression + } + + override fun replaceArgument(old: Expression, new: Expression): Boolean { + // Not supported, too complex + return false + } + + override fun typeChanged(newType: Type, src: HasType) { + // Normally, we would check, if the source comes from our initializers, but we want to limit + // the iteration of the initializer list (which can potentially contain tens of thousands of + // entries in generated code), we skip it here. + // + // So we just have to look what kind of object we are initializing (its type is stored in + // our "type), to see whether we need to propagate something at all. If it has an array + // type, we need to propagate an array version of the incoming type. If our "target" is a + // regular object type, we do NOT propagate anything at all, because in this case we get the + // types of individual fields, and we are not interested in those (yet). + val type = type + if (type is PointerType && type.pointerOrigin == PointerType.PointerOrigin.ARRAY) { + addAssignedType(newType.array()) + } + } + + override fun assignedTypeChanged(assignedTypes: Set, src: HasType) { + // Same as above, we can just propagate the incoming assigned types to us (in array form), + // if we are initializing an array + val type = type + if (type is PointerType && type.pointerOrigin == PointerType.PointerOrigin.ARRAY) { + addAssignedTypes(assignedTypes.map { it.array() }.toSet()) + } + } + override fun equals(other: Any?): Boolean { if (this === other) return true if (other !is InitializerListExpression) return false diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/LambdaExpression.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/LambdaExpression.kt index 08a1fd0bbd..a737cf20f1 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/LambdaExpression.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/LambdaExpression.kt @@ -26,19 +26,18 @@ package de.fraunhofer.aisec.cpg.graph.statements.expressions import de.fraunhofer.aisec.cpg.graph.AST -import de.fraunhofer.aisec.cpg.graph.HasType import de.fraunhofer.aisec.cpg.graph.declarations.FunctionDeclaration import de.fraunhofer.aisec.cpg.graph.declarations.ValueDeclaration -import de.fraunhofer.aisec.cpg.graph.isTypeSystemActive -import de.fraunhofer.aisec.cpg.graph.types.FunctionPointerType +import de.fraunhofer.aisec.cpg.graph.pointer +import de.fraunhofer.aisec.cpg.graph.types.FunctionType +import de.fraunhofer.aisec.cpg.graph.types.HasType import de.fraunhofer.aisec.cpg.graph.types.Type -import de.fraunhofer.aisec.cpg.graph.types.UnknownType /** * This expression denotes the usage of an anonymous / lambda function. It connects the inner * anonymous function to the user of a lambda function with an expression. */ -class LambdaExpression : Expression(), HasType.TypeListener { +class LambdaExpression : Expression(), HasType.TypeObserver { /** * If [areVariablesMutable] is false, only the (outer) variables in this list can be modified @@ -52,46 +51,25 @@ class LambdaExpression : Expression(), HasType.TypeListener { @AST var function: FunctionDeclaration? = null set(value) { - if (value != null) { - value.unregisterTypeListener(this) - if (value is HasType.TypeListener) { - unregisterTypeListener(value) - } - } + value?.unregisterTypeObserver(this) field = value - value?.registerTypeListener(this) + value?.registerTypeObserver(this) } - override fun typeChanged(src: HasType, root: MutableList, oldType: Type) { - if (!isTypeSystemActive) { + override fun typeChanged(newType: Type, src: HasType) { + // Make sure our src is the function + if (src != function) { return } - if (type !is UnknownType && src.propagationType == oldType) { - return - } - - if (src !is FunctionDeclaration) { - return - } - - val previous = type - - val parameterTypes = src.parameters.map { it.type } - val returnType = src.propagationType - - // the incoming "type" is associated to the function and it is only its return type (if it - // is known). what we really want is to construct a function type, or rather a function - // pointer type, since this is the closest to what we have - val functionType = FunctionPointerType(parameterTypes, this.language, returnType) - - setType(functionType, root) - if (previous != type) { - type.typeOrigin = Type.Origin.DATAFLOW + // We should only propagate a function type, coming from our declared function + if (newType is FunctionType) { + // Propagate a pointer reference to the function + this.type = newType.pointer() } } - override fun possibleSubTypesChanged(src: HasType, root: MutableList) { - // do not take sub types from the listener + override fun assignedTypeChanged(assignedTypes: Set, src: HasType) { + // Nothing to do } } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/MemberCallExpression.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/MemberCallExpression.kt index 239d152845..588be27338 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/MemberCallExpression.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/MemberCallExpression.kt @@ -38,7 +38,7 @@ import java.util.* * While this node implements [HasBase], this is basically just a shortcut to access the base of the * underlying [callee] property, if appropriate. */ -class MemberCallExpression : CallExpression(), HasBase { +class MemberCallExpression : CallExpression(), HasBase, HasOperatorCode { /** * The base object. This is basically a shortcut to accessing the base of the [callee], if it * has one (i.e., if it implements [HasBase]). This is the case for example, if it is a diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/MemberExpression.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/MemberExpression.kt index 5753976c99..59c9588064 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/MemberExpression.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/MemberExpression.kt @@ -27,9 +27,9 @@ package de.fraunhofer.aisec.cpg.graph.statements.expressions import de.fraunhofer.aisec.cpg.graph.AST import de.fraunhofer.aisec.cpg.graph.HasBase -import de.fraunhofer.aisec.cpg.graph.HasType import de.fraunhofer.aisec.cpg.graph.declarations.RecordDeclaration import de.fraunhofer.aisec.cpg.graph.fqn +import de.fraunhofer.aisec.cpg.graph.types.HasType import de.fraunhofer.aisec.cpg.graph.types.Type import java.util.Objects import org.apache.commons.lang3.builder.ToStringBuilder @@ -43,10 +43,10 @@ class MemberExpression : DeclaredReferenceExpression(), HasBase { @AST override var base: Expression = ProblemExpression("could not parse base expression") set(value) { - field.unregisterTypeListener(this) + field.unregisterTypeObserver(this) field = value updateName() - value.registerTypeListener(this) + value.registerTypeObserver(this) } override var operatorCode: String? = null @@ -58,22 +58,6 @@ class MemberExpression : DeclaredReferenceExpression(), HasBase { .toString() } - override fun typeChanged(src: HasType, root: MutableList, oldType: Type) { - // We are basically only interested in type changes from our base to update the naming. We - // need to ignore actual changes to the type because otherwise things go horribly wrong - if (src == base) { - updateName() - } else { - super.typeChanged(src, root, oldType) - } - } - - override fun possibleSubTypesChanged(src: HasType, root: MutableList) { - if (src != base) { - super.possibleSubTypesChanged(src, root) - } - } - override fun equals(other: Any?): Boolean { if (this === other) return true if (other !is MemberExpression) return false @@ -82,6 +66,16 @@ class MemberExpression : DeclaredReferenceExpression(), HasBase { override fun hashCode() = Objects.hash(super.hashCode(), base) + override fun typeChanged(newType: Type, src: HasType) { + // We are basically only interested in type changes from our base to update the naming. We + // need to ignore actual changes to the type because otherwise things go horribly wrong + if (src == base) { + updateName() + } else { + super.typeChanged(newType, src) + } + } + private fun updateName() { this.name = base.type.root.name.fqn(name.localName) } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/UnaryOperator.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/UnaryOperator.kt index be71e58f86..fca3d2deda 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/UnaryOperator.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/UnaryOperator.kt @@ -27,24 +27,20 @@ package de.fraunhofer.aisec.cpg.graph.statements.expressions import de.fraunhofer.aisec.cpg.graph.AST import de.fraunhofer.aisec.cpg.graph.AccessValues -import de.fraunhofer.aisec.cpg.graph.HasType -import de.fraunhofer.aisec.cpg.graph.isTypeSystemActive -import de.fraunhofer.aisec.cpg.graph.types.PointerType +import de.fraunhofer.aisec.cpg.graph.pointer +import de.fraunhofer.aisec.cpg.graph.types.HasType import de.fraunhofer.aisec.cpg.graph.types.Type -import de.fraunhofer.aisec.cpg.helpers.Util.distinctBy -import java.util.stream.Collectors import org.apache.commons.lang3.builder.ToStringBuilder -import org.neo4j.ogm.annotation.Transient /** A unary operator expression, involving one expression and an operator, such as `a++`. */ -class UnaryOperator : Expression(), HasType.TypeListener { +class UnaryOperator : Expression(), HasType.TypeObserver { /** The expression on which the operation is applied. */ @AST var input: Expression = ProblemExpression("could not parse input") set(value) { - field.unregisterTypeListener(this) + field.unregisterTypeObserver(this) field = value - input.registerTypeListener(this) + input.registerTypeObserver(this) changeExpressionAccess() } @@ -61,8 +57,6 @@ class UnaryOperator : Expression(), HasType.TypeListener { /** Specifies, whether this a pre fix operation. */ var isPrefix = false - @Transient private val checked: MutableList = ArrayList() - private fun changeExpressionAccess() { var access = AccessValues.READ if (operatorCode == "++" || operatorCode == "--") { @@ -73,103 +67,49 @@ class UnaryOperator : Expression(), HasType.TypeListener { } } - private fun getsDataFromInput( - curr: HasType.TypeListener, - target: HasType.TypeListener - ): Boolean { - val worklist: MutableList = ArrayList() - worklist.add(curr) - while (worklist.isNotEmpty()) { - val tl = worklist.removeAt(0) - if (!checked.contains(tl)) { - checked.add(tl) - if (tl === target) { - return true - } - if (curr is HasType) { - worklist.addAll((curr as HasType).typeListeners) - } - } - } - return false - } - - private fun getsDataFromInput(listener: HasType.TypeListener): Boolean { - checked.clear() - for (l in input.typeListeners) { - if (getsDataFromInput(l, listener)) return true - } - return false + override fun toString(): String { + return ToStringBuilder(this, TO_STRING_STYLE) + .appendSuper(super.toString()) + .append("operatorCode", operatorCode) + .append("postfix", isPostfix) + .append("prefix", isPrefix) + .toString() } - override fun typeChanged(src: HasType, root: MutableList, oldType: Type) { - if (!isTypeSystemActive) { + override fun typeChanged(newType: Type, src: HasType) { + // Only accept type changes from out input + if (src != input) { return } - val previous = type - if (src === input) { - var newType = src.propagationType - if (operatorCode == "*") { - newType = newType.dereference() - } else if (operatorCode == "&") { - newType = newType.reference(PointerType.PointerOrigin.POINTER) - } - setType(newType, root) - } else { - // Our input didn't change, so we don't need to (de)reference the type - setType(src.propagationType, root) - // Pass the type on to the input in an inversely (de)referenced way - var newType: Type? = src.propagationType - if (operatorCode == "*") { - newType = src.propagationType.reference(PointerType.PointerOrigin.POINTER) - } else if (operatorCode == "&") { - newType = src.propagationType.dereference() + val type = + when (operatorCode) { + "*" -> newType.dereference() + "&" -> newType.pointer() + else -> newType } - newType?.let { input.setType(it, mutableListOf(this)) } - } - if (previous != type) { - type.typeOrigin = Type.Origin.DATAFLOW - } + this.type = type } - override fun possibleSubTypesChanged(src: HasType, root: MutableList) { - if (!isTypeSystemActive) { - return - } - if (src is HasType.TypeListener && getsDataFromInput(src as HasType.TypeListener)) { + override fun assignedTypeChanged(assignedTypes: Set, src: HasType) { + // Only accept type changes from out input + if (src != input) { return } - var currSubTypes: MutableList = ArrayList(possibleSubTypes) - val newSubTypes = src.possibleSubTypes - currSubTypes.addAll(newSubTypes) - if (operatorCode == "*") { - currSubTypes = - currSubTypes - .stream() - .filter(distinctBy { obj: Type -> obj.typeName }) - .map { obj: Type -> obj.dereference() } - .collect(Collectors.toList()) - } else if (operatorCode == "&") { - currSubTypes = - currSubTypes - .stream() - .filter(distinctBy { obj: Type -> obj.typeName }) - .map { t: Type -> t.reference(PointerType.PointerOrigin.POINTER) } - .collect(Collectors.toList()) - } - _possibleSubTypes.clear() - setPossibleSubTypes(currSubTypes, root) // notify about the new type - } - override fun toString(): String { - return ToStringBuilder(this, TO_STRING_STYLE) - .appendSuper(super.toString()) - .append("operatorCode", operatorCode) - .append("postfix", isPostfix) - .append("prefix", isPrefix) - .toString() + // Apply our operator to all assigned types and forward them to us + this.addAssignedTypes( + assignedTypes + .map { + when (operatorCode) { + "*" -> it.dereference() + "&" -> it.pointer() + else -> it + } + } + .toSet() + ) } override fun equals(other: Any?): Boolean { diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/AutoType.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/AutoType.kt new file mode 100644 index 0000000000..5d98db1138 --- /dev/null +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/AutoType.kt @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2023, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.graph.types + +import de.fraunhofer.aisec.cpg.frontends.Language +import de.fraunhofer.aisec.cpg.graph.unknownType + +/** + * This type represents a [Type] that uses auto-inference (usually from an initializer) to determine + * it's actual type. It is commonly used in dynamically typed languages or in languages that have a + * special keyword, such as `auto` in C++. + * + * Note: This is intentionally a distinct type and not the [UnknownType]. + */ +class AutoType(override var language: Language<*>?) : Type("auto", language) { + override fun reference(pointer: PointerType.PointerOrigin?): Type { + return PointerType(this, pointer) + } + + override fun dereference(): Type { + return unknownType() + } + + override fun duplicate(): Type { + return AutoType(this.language) + } +} diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/FunctionType.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/FunctionType.kt index 0dda94a841..d228edc6c6 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/FunctionType.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/FunctionType.kt @@ -27,7 +27,6 @@ package de.fraunhofer.aisec.cpg.graph.types import de.fraunhofer.aisec.cpg.frontends.Language import de.fraunhofer.aisec.cpg.graph.declarations.FunctionDeclaration -import de.fraunhofer.aisec.cpg.graph.registerType import de.fraunhofer.aisec.cpg.graph.unknownType /** diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/HasType.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/HasType.kt index edf680fc7c..8df00cad74 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/HasType.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/HasType.kt @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019, Fraunhofer AISEC. All rights reserved. + * Copyright (c) 2023, Fraunhofer AISEC. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -23,109 +23,174 @@ * \______/ \__| \______/ * */ -package de.fraunhofer.aisec.cpg.graph +package de.fraunhofer.aisec.cpg.graph.types -import de.fraunhofer.aisec.cpg.TypeManager import de.fraunhofer.aisec.cpg.frontends.TranslationException -import de.fraunhofer.aisec.cpg.graph.types.Type -import java.util.Optional - -interface HasType : ContextProvider { - var type: Type - - /** - * @return The returned Type is always the same as getType() with the exception of ReferenceType - * since there is no case in which we want to propagate a reference when using typeChanged() - */ - val propagationType: Type +import de.fraunhofer.aisec.cpg.graph.ContextProvider +import de.fraunhofer.aisec.cpg.graph.LanguageProvider +import de.fraunhofer.aisec.cpg.graph.Node +import de.fraunhofer.aisec.cpg.graph.declarations.ValueDeclaration +import de.fraunhofer.aisec.cpg.graph.statements.expressions.DeclaredReferenceExpression +import de.fraunhofer.aisec.cpg.graph.statements.expressions.Expression +import de.fraunhofer.aisec.cpg.graph.statements.expressions.Literal +import de.fraunhofer.aisec.cpg.graph.statements.expressions.UnaryOperator +import java.util.* + +/** + * This interfaces denotes that the given [Node] has a "type". Currently, we only have two known + * implementations of this class, an [Expression] and a [ValueDeclaration]. All other nodes with + * types should derive from these two base classes. + */ +interface HasType : ContextProvider, LanguageProvider { /** - * Side-effect free type modification WARNING: This should only be used by the TypeSystem Pass + * This property refers to the *definite* [Type] that the [Node] has. If you are unsure about + * what it's type is, you should prefer to set it to the [UnknownType]. It is usually one of the + * following: + * - the type declared by the [Node], e.g., by a [ValueDeclaration] + * - intrinsically tied to the node, e.g. an [IntegerType] in an integer [Literal] + * - the [Type] of a declaration a node is referring to, e.g., in a + * [DeclaredReferenceExpression] * - * @param type new type + * An implementation of this must be sure to invoke [informObservers]. */ - fun updateType(type: Type) - - fun updatePossibleSubtypes(types: List) + var type: Type /** - * Set the node's type. This may start a chain of type listener notifications + * This property refers to a list of [Type] nodes which are assigned to that [Node]. This could + * be different from the [HasType.type]. A common example is that a node could contain an + * interface as a [HasType.type], but the actual implementation of the type as one of the + * [assignedTypes]. This could potentially also be empty, if we don't see any assignments to + * this expression. * - * @param type new type - * @param root The nodes which we have seen in the type change chain. When a node receives a - * type setting command where root.contains(this), we know that we have a type listener circle - * and can abort. If root is an empty list, the type change is seen as an externally triggered - * event and subsequent type listeners receive the current node as their root. + * Note: in order to properly inform observers, one should NOT use the regular [MutableSet.add] + * or [MutableSet.addAll] but rather use [addAssignedType] and [addAssignedTypes]. Otherwise, we + * cannot watch for changes within the set. We therefore only expose this as a [Set], but an + * implementing class MUST implement this as a [MutableSet] so that we can modify it internally. */ - fun setType(type: Type, root: MutableList?) - - var possibleSubTypes: List + var assignedTypes: Set /** - * Set the node's possible subtypes. Listener circle detection works the same way as with - * [ ][.setType] - * - * @param possibleSubTypes the set of possible sub types - * @param root A list of already seen nodes which is used for detecting loops. + * Adds [type] to the list of [HasType.assignedTypes] and informs all observers about the + * change. */ - fun setPossibleSubTypes(possibleSubTypes: List, root: MutableList) - - val typeListeners: MutableSet - - fun registerTypeListener(listener: TypeListener) { - val root = mutableListOf(this) - typeListeners.add(listener) - listener.typeChanged(this, root, type) - listener.possibleSubTypesChanged(this, root) + fun addAssignedType(type: Type) { + if (language?.shouldPropagateType(this, type) == false) { + return + } + + val changed = (this.assignedTypes as MutableSet).add(type) + if (changed) { + informObservers(TypeObserver.ChangeType.ASSIGNED_TYPE) + } } - fun unregisterTypeListener(listener: TypeListener) { - typeListeners.remove(listener) + /** + * Adds all [types] to the list of [HasType.assignedTypes] and informs all observers about the + * change. + */ + fun addAssignedTypes(types: Set) { + val changed = + (this.assignedTypes as MutableSet).addAll( + types.filter { language?.shouldPropagateType(this, it) == true } + ) + if (changed) { + informObservers(TypeObserver.ChangeType.ASSIGNED_TYPE) + } } - fun refreshType() - /** - * Used to set the type and clear the possible subtypes list for when a type is more precise - * than the current. - * - * @param type the more precise type + * A list of [TypeObserver] objects that will be informed about type changes, usually by + * [informObservers]. */ - fun resetTypes(type: Type) + val typeObservers: MutableList - interface TypeListener { - fun typeChanged(src: HasType, root: MutableList, oldType: Type) + /** + * A [TypeObserver] can be used by its implementing class to observe changes to the + * [HasType.type] and/or [HasType.assignedTypes] of a [Node] (that implements [HasType]). The + * implementing node can then decide if and how to propagate this type information to itself + * (and possibly to others). Examples include modifying the incoming type depending on an + * operator, e.g., in a [UnaryOperator] expression. Changes to [HasType.type] will invoke + * [typeChanged], changes to [HasType.assignedTypes] will invoke [assignedTypes]. + */ + interface TypeObserver { + enum class ChangeType { + TYPE, + ASSIGNED_TYPE + } + + /** + * This callback function will be invoked, if the observed node changes its [HasType.type]. + */ + fun typeChanged(newType: Type, src: HasType) + + /** + * This callback function will be invoked, if the observed node changes its + * [HasType.assignedTypes]. + */ + fun assignedTypeChanged(assignedTypes: Set, src: HasType) + } - fun possibleSubTypesChanged(src: HasType, root: MutableList) + /** + * This function SHOULD be used be an implementing class to inform observers about type changes. + * While the implementing class can technically do this on its own, it is strongly recommended + * to use this function to harmonize the behaviour of propagating types. + */ + fun informObservers(changeType: TypeObserver.ChangeType) { + if (changeType == TypeObserver.ChangeType.ASSIGNED_TYPE) { + val assignedTypes = this.assignedTypes + if (assignedTypes.isEmpty()) { + return + } + // Inform all type observers about the changes + for (observer in typeObservers) { + observer.assignedTypeChanged(assignedTypes, this) + } + } else { + val newType = this.type + if (newType is UnknownType) { + return + } + // Inform all type observers about the changes + for (observer in typeObservers) { + observer.typeChanged(newType, this) + } + } } /** - * The Typeresolver needs to be aware of all outgoing edges to types in order to merge equal - * types to the same node. For the primary type edge, this is achieved through the hasType - * interface. If a node has additional type edges (e.g. default type in [ ]) the node must - * implement the updateType method, so that the current type is always replaced with the merged - * one + * Registers the given [typeObservers] to be informed about type updates. This also immediately + * invokes both [TypeObserver.typeChanged] and [TypeObserver.assignedTypeChanged]. */ - interface SecondaryTypeEdge { - fun updateType(typeState: Collection) + fun registerTypeObserver(typeObserver: TypeObserver) { + typeObservers += typeObserver + + // If we would only propagate the unknown type, we can also skip it + val newType = this.type + if (newType !is UnknownType) { + // Immediately inform about changes + typeObserver.typeChanged(newType, this) + } + + // If we would propagate an empty list, we can also skip it + val assignedTypes = this.assignedTypes + if (assignedTypes.isNotEmpty()) { + // Immediately inform about changes + typeObserver.assignedTypeChanged(assignedTypes, this) + } } -} -val Node.isTypeSystemActive: Boolean - get() { - return TypeManager.isTypeSystemActive + /** Unregisters the given [typeObservers] from the list of observers. */ + fun unregisterTypeObserver(typeObserver: TypeObserver) { + typeObservers -= typeObserver } +} fun Node.isSupertypeOf(superType: Type, subType: Type): Boolean { val c = ctx ?: throw TranslationException("context not available") return c.typeManager.isSupertypeOf(superType, subType, this) } -fun HasType.cacheType(type: Type) { - val c = ctx ?: throw TranslationException("context not available") - c.typeManager.cacheType(this, type) -} - fun Node.registerType(type: T): T { val c = ctx ?: throw TranslationException("context not available") return c.typeManager.registerType(type) @@ -135,8 +200,3 @@ fun Node.getCommonType(types: Collection): Optional { val c = ctx ?: throw TranslationException("context not available") return c.typeManager.getCommonType(types, this.ctx) } - -fun Node.stopPropagation(type: Type, newType: Type): Boolean { - val c = ctx ?: throw TranslationException("context not available") - return c.typeManager.stopPropagation(type, newType) -} diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/ObjectType.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/ObjectType.kt index d81ecc441e..bbe1d32282 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/ObjectType.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/ObjectType.kt @@ -26,7 +26,6 @@ package de.fraunhofer.aisec.cpg.graph.types import de.fraunhofer.aisec.cpg.frontends.Language -import de.fraunhofer.aisec.cpg.graph.HasType.SecondaryTypeEdge import de.fraunhofer.aisec.cpg.graph.declarations.RecordDeclaration import de.fraunhofer.aisec.cpg.graph.edge.Properties import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/SecondaryTypeEdge.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/SecondaryTypeEdge.kt new file mode 100644 index 0000000000..c0519df0d6 --- /dev/null +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/SecondaryTypeEdge.kt @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2023, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.graph.types + +import de.fraunhofer.aisec.cpg.graph.declarations.TypeParamDeclaration +import de.fraunhofer.aisec.cpg.passes.TypeResolver + +/** + * The [TypeResolver] needs to be aware of all outgoing edges to types in order to merge equal types + * to the same node. For the primary type edge, this is achieved through the hasType interface. If a + * node has additional type edges (e.g. [TypeParamDeclaration.default]) the node must implement the + * [updateType] method, so that the current type is always replaced with the merged one + */ +interface SecondaryTypeEdge { + fun updateType(typeState: Collection) +} diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/Type.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/Type.kt index a43599c030..4ebd21cef4 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/Type.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/Type.kt @@ -159,8 +159,10 @@ abstract class Type : Node { */ get() = (this is ObjectType || + this is AutoType || this is UnknownType || this is FunctionType || + this is ProblemType || this is TupleType // TODO(oxisto): convert FunctionPointerType to second order type || this is FunctionPointerType || diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/UnknownType.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/UnknownType.kt index ea8925e75f..57b2c678fc 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/UnknownType.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/types/UnknownType.kt @@ -32,9 +32,9 @@ import java.util.* import java.util.concurrent.ConcurrentHashMap /** - * UnknownType describe the case in which it is not possible for the CPG to determine which Type is - * used. E.g.: This occurs when the type is inferred by the compiler automatically when using - * keywords such as auto in cpp + * UnknownType describe the case in which it is not possible for the CPG to determine which [Type] + * is used. Ideally, this type is only assigned temporary and then later replaced with an actual + * known type. But, because we sometimes do fuzzy parsing, this might not be the case all the time. */ class UnknownType private constructor() : Type() { init { diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/helpers/SubgraphWalker.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/helpers/SubgraphWalker.kt index a86ca0329a..38d3a66fa1 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/helpers/SubgraphWalker.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/helpers/SubgraphWalker.kt @@ -28,7 +28,6 @@ package de.fraunhofer.aisec.cpg.helpers import de.fraunhofer.aisec.cpg.ScopeManager import de.fraunhofer.aisec.cpg.frontends.LanguageFrontend import de.fraunhofer.aisec.cpg.graph.AST -import de.fraunhofer.aisec.cpg.graph.HasType import de.fraunhofer.aisec.cpg.graph.Node import de.fraunhofer.aisec.cpg.graph.declarations.FunctionDeclaration import de.fraunhofer.aisec.cpg.graph.declarations.RecordDeclaration @@ -38,7 +37,6 @@ import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge.Companion.checkForPropertyEdge import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge.Companion.unwrap import de.fraunhofer.aisec.cpg.graph.statements.CompoundStatement -import de.fraunhofer.aisec.cpg.processing.IVisitor import de.fraunhofer.aisec.cpg.processing.strategy.Strategy import java.lang.annotation.AnnotationFormatError import java.lang.reflect.Field @@ -250,20 +248,6 @@ object SubgraphWalker { return border } - fun refreshType(node: Node) { - // Using a visitor to avoid loops in the AST - node.accept( - Strategy::AST_FORWARD, - object : IVisitor() { - override fun visit(t: Node) { - if (t is HasType) { - (t as HasType).refreshType() - } - } - } - ) - } - /** * For better readability: `result.entries` instead of `result.get(0)` when working with * getEOGPathEdges. Can be used for all subgraphs in subgraphs, e.g. AST entries and exits in a diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/CXXCallResolverHelper.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/CXXCallResolverHelper.kt index 1030524b39..ed9a77e044 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/CXXCallResolverHelper.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/CXXCallResolverHelper.kt @@ -30,10 +30,7 @@ import de.fraunhofer.aisec.cpg.TypeManager import de.fraunhofer.aisec.cpg.graph.* import de.fraunhofer.aisec.cpg.graph.declarations.* import de.fraunhofer.aisec.cpg.graph.statements.expressions.* -import de.fraunhofer.aisec.cpg.graph.types.ObjectType -import de.fraunhofer.aisec.cpg.graph.types.ParameterizedType -import de.fraunhofer.aisec.cpg.graph.types.Type -import de.fraunhofer.aisec.cpg.graph.types.UnknownType +import de.fraunhofer.aisec.cpg.graph.types.* import java.util.HashMap import java.util.regex.Pattern @@ -44,19 +41,15 @@ import java.util.regex.Pattern * signature by means of casting */ fun compatibleSignatures( + call: CallExpression, callSignature: List, - functionSignature: List, - ctx: TranslationContext + functionSignature: List ): Boolean { return if (callSignature.size == functionSignature.size) { for (i in callSignature.indices) { if ( callSignature[i].isPrimitive != functionSignature[i].isPrimitive && - !ctx.typeManager.isSupertypeOf( - functionSignature[i], - callSignature[i], - ctx.scopeManager - ) + !call.isSupertypeOf(functionSignature[i], callSignature[i]) ) { return false } @@ -97,8 +90,7 @@ fun getCallSignatureWithDefaults( */ fun resolveWithImplicitCast( call: CallExpression, - initialInvocationCandidates: List, - ctx: TranslationContext + initialInvocationCandidates: List ): List { // Output list for invocationTargets obtaining a valid signature by performing implicit @@ -111,13 +103,13 @@ fun resolveWithImplicitCast( for (functionDeclaration in initialInvocationCandidates) { val callSignature = getCallSignatureWithDefaults(call, functionDeclaration) // Check if the signatures match by implicit casts - if (compatibleSignatures(callSignature, functionDeclaration.signatureTypes, ctx)) { + if (compatibleSignatures(call, callSignature, functionDeclaration.signatureTypes)) { val implicitCastTargets = signatureWithImplicitCastTransformation( + call, getCallSignatureWithDefaults(call, functionDeclaration), call.arguments, functionDeclaration.signatureTypes, - ctx ) if (implicitCasts == null) { implicitCasts = implicitCastTargets @@ -126,7 +118,7 @@ fun resolveWithImplicitCast( // to the same target type checkMostCommonImplicitCast(implicitCasts, implicitCastTargets) } - if (compatibleSignatures(call.signature, functionDeclaration.signatureTypes, ctx)) { + if (compatibleSignatures(call, call.signature, functionDeclaration.signatureTypes)) { invocationTargetsWithImplicitCast.add(functionDeclaration) } else { invocationTargetsWithImplicitCastAndDefaults.add(functionDeclaration) @@ -262,8 +254,7 @@ fun shouldContinueSearchInParent(recordDeclaration: RecordDeclaration?, name: St */ fun resolveConstructorWithImplicitCast( constructExpression: ConstructExpression, - recordDeclaration: RecordDeclaration, - ctx: TranslationContext + recordDeclaration: RecordDeclaration ): ConstructorDeclaration? { for (constructorDeclaration in recordDeclaration.constructors) { val workingSignature = mutableListOf(*constructExpression.signature.toTypedArray()) @@ -278,29 +269,33 @@ fun resolveConstructorWithImplicitCast( } if ( compatibleSignatures( + constructExpression, constructExpression.signature, constructorDeclaration.signatureTypes, - ctx ) ) { val implicitCasts = signatureWithImplicitCastTransformation( + constructExpression, constructExpression.signature, constructExpression.arguments, constructorDeclaration.signatureTypes, - ctx ) applyImplicitCastToArguments(constructExpression, implicitCasts) return constructorDeclaration } else if ( - compatibleSignatures(workingSignature, constructorDeclaration.signatureTypes, ctx) + compatibleSignatures( + constructExpression, + workingSignature, + constructorDeclaration.signatureTypes, + ) ) { val implicitCasts = signatureWithImplicitCastTransformation( + constructExpression, getCallSignatureWithDefaults(constructExpression, constructorDeclaration), constructExpression.arguments, constructorDeclaration.signatureTypes, - ctx ) applyImplicitCastToArguments(constructExpression, implicitCasts) return constructorDeclaration @@ -362,10 +357,10 @@ fun applyTemplateInstantiation( val templateCallSignature = templateCall.signature val callSignatureImplicit = signatureWithImplicitCastTransformation( + templateCall, templateCallSignature, templateCall.arguments, templateFunctionSignature, - ctx ) for (i in callSignatureImplicit.indices) { val cast = callSignatureImplicit[i] @@ -399,10 +394,10 @@ fun applyTemplateInstantiation( * FunctionDeclaration cannot be reached through implicit casts */ fun signatureWithImplicitCastTransformation( + call: CallExpression, callSignature: List, arguments: List, functionSignature: List, - ctx: TranslationContext ): MutableList { val implicitCasts = mutableListOf() if (callSignature.size != functionSignature.size) return implicitCasts @@ -412,7 +407,7 @@ fun signatureWithImplicitCastTransformation( val funcType = functionSignature[i] if (callType?.isPrimitive == true && funcType.isPrimitive && callType != funcType) { val implicitCast = CastExpression() - implicitCast.ctx = ctx + implicitCast.ctx = call.ctx implicitCast.isImplicit = true implicitCast.castType = funcType implicitCast.language = funcType.language diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/CallResolver.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/CallResolver.kt index 70b17a789b..5bd0a46fa2 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/CallResolver.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/CallResolver.kt @@ -110,6 +110,7 @@ open class CallResolver(ctx: TranslationContext) : SymbolResolverPass(ctx) { val currInitializer = node.initializer if (currInitializer == null && node.isImplicitInitializerAllowed) { val initializer = node.newConstructExpression(typeString, "$typeString()") + initializer.type = node.type initializer.isImplicit = true node.initializer = initializer node.templateParameters?.let { @@ -125,6 +126,7 @@ open class CallResolver(ctx: TranslationContext) : SymbolResolverPass(ctx) { val signature = arguments.map(Node::code).joinToString(", ") val initializer = node.newConstructExpression(typeString, "$typeString($signature)") + initializer.type = node.type initializer.arguments = mutableListOf(*arguments.toTypedArray()) initializer.isImplicit = true node.initializer = initializer @@ -483,7 +485,7 @@ open class CallResolver(ctx: TranslationContext) : SymbolResolverPass(ctx) { if (node is MemberCallExpression) { node.base?.let { base -> possibleTypes.add(base.type) - possibleTypes.addAll(base.possibleSubTypes) + possibleTypes.addAll(base.assignedTypes) } } else { // This could be a C++ member call with an implicit this (which we do not create), so @@ -607,7 +609,7 @@ open class CallResolver(ctx: TranslationContext) : SymbolResolverPass(ctx) { // If we don't find any candidate and our current language is c/c++ we check if there is // a candidate with an implicit cast constructorCandidate = - resolveConstructorWithImplicitCast(constructExpression, recordDeclaration, ctx) + resolveConstructorWithImplicitCast(constructExpression, recordDeclaration) } return constructorCandidate diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/ControlFlowSensitiveDFGPass.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/ControlFlowSensitiveDFGPass.kt index 9182f2c19e..d39bfc207e 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/ControlFlowSensitiveDFGPass.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/ControlFlowSensitiveDFGPass.kt @@ -34,6 +34,8 @@ import de.fraunhofer.aisec.cpg.graph.statements.* import de.fraunhofer.aisec.cpg.graph.statements.expressions.* import de.fraunhofer.aisec.cpg.helpers.* import de.fraunhofer.aisec.cpg.passes.order.DependsOn +import kotlin.contracts.ExperimentalContracts +import kotlin.contracts.contract /** * This pass determines the data flows of DeclaredReferenceExpressions which refer to a @@ -41,6 +43,7 @@ import de.fraunhofer.aisec.cpg.passes.order.DependsOn * path, only such data flows are left which can occur when following the control flow (in terms of * the EOG) of the program. */ +@OptIn(ExperimentalContracts::class) @DependsOn(EvaluationOrderGraphPass::class) @DependsOn(DFGPass::class) open class ControlFlowSensitiveDFGPass(ctx: TranslationContext) : TranslationUnitPass(ctx) { @@ -86,7 +89,7 @@ open class ControlFlowSensitiveDFGPass(ctx: TranslationContext) : TranslationUni * code [node]. */ protected fun clearFlowsOfVariableDeclarations(node: Node) { - for (varDecl in node.variables) { + for (varDecl in node.variables.filter { it !is FieldDeclaration }) { varDecl.clearPrevDFG() varDecl.clearNextDFG() } @@ -107,7 +110,7 @@ open class ControlFlowSensitiveDFGPass(ctx: TranslationContext) : TranslationUni worklist: Worklist, Node, Set> ): State> { // We will set this if we write to a variable - val writtenDecl: Declaration? + var writtenDecl: Declaration? val currentNode = currentEdge.end val doubleState = state as DFGPassState @@ -120,7 +123,7 @@ open class ControlFlowSensitiveDFGPass(ctx: TranslationContext) : TranslationUni state.push(currentNode, PowersetLattice(setOf(initializer))) doubleState.pushToDeclarationsState(currentNode, PowersetLattice(setOf(currentNode))) - } else if (currentNode is AssignExpression) { + } else if (isSimpleAssignment(currentNode)) { // It's an assignment which can have one or multiple things on the lhs and on the // rhs. The lhs could be a declaration or a reference (or multiple of these things). // The rhs can be anything. The rhs flows to the respective lhs. To identify the @@ -145,33 +148,25 @@ open class ControlFlowSensitiveDFGPass(ctx: TranslationContext) : TranslationUni state.push(input, doubleState.declarationsState[writtenDecl]) doubleState.declarationsState[writtenDecl] = PowersetLattice(setOf(input)) } - } else if (isSimpleAssignment(currentNode)) { - // Only the lhs is the last write statement here and the variable which is written - // to. - writtenDecl = - ((currentNode as BinaryOperator).lhs as DeclaredReferenceExpression).refersTo - - if (writtenDecl != null) { - doubleState.declarationsState[writtenDecl] = PowersetLattice(setOf(currentNode.lhs)) - } } else if (isCompoundAssignment(currentNode)) { // We write to the lhs, but it also serves as an input => We first get all previous // writes to the lhs and then add the flow from lhs and rhs to the current node. // The write operation goes to the variable in the lhs - writtenDecl = - ((currentNode as BinaryOperator).lhs as? DeclaredReferenceExpression)?.refersTo + val lhs = currentNode.lhs.singleOrNull() + writtenDecl = (lhs as? DeclaredReferenceExpression)?.refersTo - if (writtenDecl != null) { + if (writtenDecl != null && lhs != null) { // Data flows from the last writes to the lhs variable to this node - state.push(currentNode.lhs, doubleState.declarationsState[writtenDecl]) + state.push(lhs, doubleState.declarationsState[writtenDecl]) // The whole current node is the place of the last update, not (only) the lhs! - doubleState.declarationsState[writtenDecl] = PowersetLattice(setOf(currentNode.lhs)) + doubleState.declarationsState[writtenDecl] = PowersetLattice(setOf(lhs)) } } else if ( (currentNode as? DeclaredReferenceExpression)?.access == AccessValues.READ && - currentNode.refersTo is VariableDeclaration + currentNode.refersTo is VariableDeclaration && + currentNode.refersTo !is FieldDeclaration ) { // We can only find a change if there's a state for the variable doubleState.declarationsState[currentNode.refersTo]?.let { @@ -245,17 +240,18 @@ open class ControlFlowSensitiveDFGPass(ctx: TranslationContext) : TranslationUni * Checks if the node performs an operation and an assignment at the same time e.g. with the * operators +=, -=, *=, ... */ - protected fun isCompoundAssignment(currentNode: Node) = - currentNode is BinaryOperator && + protected fun isCompoundAssignment(currentNode: Node): Boolean { + contract { returns(true) implies (currentNode is AssignExpression) } + return currentNode is AssignExpression && currentNode.operatorCode in (currentNode.language?.compoundAssignmentOperators ?: setOf()) && - (currentNode.lhs as? DeclaredReferenceExpression)?.refersTo != null + (currentNode.lhs.singleOrNull() as? DeclaredReferenceExpression)?.refersTo != null + } - /** Checks if the node is a simple assignment of the form `var = ...` */ - protected fun isSimpleAssignment(currentNode: Node) = - currentNode is BinaryOperator && - currentNode.operatorCode == "=" && - (currentNode.lhs as? DeclaredReferenceExpression)?.refersTo != null + protected fun isSimpleAssignment(currentNode: Node): Boolean { + contract { returns(true) implies (currentNode is AssignExpression) } + return currentNode is AssignExpression && currentNode.operatorCode == "=" + } /** Checks if the node is an increment or decrement operator (e.g. i++, i--, ++i, --i) */ protected fun isIncOrDec(currentNode: Node) = diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/DFGPass.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/DFGPass.kt index 2ca7531daa..580ebe69ed 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/DFGPass.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/DFGPass.kt @@ -71,7 +71,7 @@ class DFGPass(ctx: TranslationContext) : ComponentPass(ctx) { is NewExpression -> handleNewExpression(node) // We keep the logic for the InitializerListExpression in that class because the // performance would decrease too much. - // is InitializerListExpression -> handleInitializerListExpression(node) + is InitializerListExpression -> handleInitializerListExpression(node) is KeyValueExpression -> handleKeyValueExpression(node) is LambdaExpression -> handleLambdaExpression(node) is UnaryOperator -> handleUnaryOperator(node) @@ -272,14 +272,9 @@ class DFGPass(ctx: TranslationContext) : ComponentPass(ctx) { /** * Adds the DFG edges for an [InitializerListExpression]. All values in the initializer flow to * this expression. - * - * TODO: This change seems to have performance issues! */ protected fun handleInitializerListExpression(node: InitializerListExpression) { - node.initializers.forEach { - it.registerTypeListener(node) - node.addPrevDFG(it) - } + node.initializers.forEach { node.addPrevDFG(it) } } /** diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/TypeHierarchyResolver.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/TypeHierarchyResolver.kt index 2ab2fffb06..2e5f10af8f 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/TypeHierarchyResolver.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/TypeHierarchyResolver.kt @@ -33,7 +33,6 @@ import de.fraunhofer.aisec.cpg.graph.declarations.EnumDeclaration import de.fraunhofer.aisec.cpg.graph.declarations.MethodDeclaration import de.fraunhofer.aisec.cpg.graph.declarations.RecordDeclaration import de.fraunhofer.aisec.cpg.graph.types.Type -import de.fraunhofer.aisec.cpg.helpers.SubgraphWalker import de.fraunhofer.aisec.cpg.processing.IVisitor import de.fraunhofer.aisec.cpg.processing.strategy.Strategy import java.util.* @@ -75,8 +74,6 @@ open class TypeHierarchyResolver(ctx: TranslationContext) : ComponentPass(ctx) { directSupertypeRecords.map { findSupertypeRecords(it) }.flatten().toSet() enumDecl.superTypeDeclarations = allSupertypes } - - component.translationUnits.forEach { SubgraphWalker.refreshType(it) } } protected fun findRecordsAndEnums(node: Node) { diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/TypeResolver.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/TypeResolver.kt index b970f07d75..6e38a9a360 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/TypeResolver.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/TypeResolver.kt @@ -27,11 +27,10 @@ package de.fraunhofer.aisec.cpg.passes import de.fraunhofer.aisec.cpg.TranslationContext import de.fraunhofer.aisec.cpg.graph.Component -import de.fraunhofer.aisec.cpg.graph.HasType -import de.fraunhofer.aisec.cpg.graph.HasType.SecondaryTypeEdge import de.fraunhofer.aisec.cpg.graph.Node import de.fraunhofer.aisec.cpg.graph.declarations.RecordDeclaration import de.fraunhofer.aisec.cpg.graph.types.* +import de.fraunhofer.aisec.cpg.graph.types.SecondaryTypeEdge import de.fraunhofer.aisec.cpg.helpers.SubgraphWalker.IterativeGraphWalker import de.fraunhofer.aisec.cpg.passes.order.DependsOn @@ -181,14 +180,16 @@ open class TypeResolver(ctx: TranslationContext) : ComponentPass(ctx) { // globally unique if (node is HasType && node.type !is ParameterizedType) { val type = node.type - val types = + val newType = if (type.isFirstOrderType) { - typeState.keys - } else { - typeState.computeIfAbsent(type.root, ::mutableListOf) - } - updateType(node, types) - node.updatePossibleSubtypes(ensureUniqueSubTypes(node.possibleSubTypes)) + typeState.keys + } else { + typeState.computeIfAbsent(type.root, ::mutableListOf) + } + .firstOrNull { it == type } + if (newType != null) { + node.type = newType + } } } @@ -196,27 +197,16 @@ open class TypeResolver(ctx: TranslationContext) : ComponentPass(ctx) { * ensures that the if a nodes contains secondary type edges, those types are also merged and no * duplicate is left * - * @param node implementing [HasType.SecondaryTypeEdge] + * @param node implementing [SecondaryTypeEdge] */ protected fun ensureUniqueSecondaryTypeEdge(node: Node) { if (node is SecondaryTypeEdge) { node.updateType(typeState.keys) } else if (node is HasType && node.type is SecondaryTypeEdge) { (node.type as SecondaryTypeEdge).updateType(typeState.keys) - for (possibleSubType in node.possibleSubTypes) { - if (possibleSubType is SecondaryTypeEdge) { - possibleSubType.updateType(typeState.keys) - } - } } } - protected fun updateType(node: HasType, types: Collection) { - // TODO: Why do we perform the update only for the first type? - val typeToUpdate = types.firstOrNull { it == node.type } ?: return - node.updateType(typeToUpdate) - } - /** * Creates the recordDeclaration relationship between ObjectTypes and RecordDeclaration (from * the Type to the Class) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/VariableUsageResolver.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/VariableUsageResolver.kt index f59a594076..d37ba927fd 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/VariableUsageResolver.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/VariableUsageResolver.kt @@ -37,6 +37,7 @@ import de.fraunhofer.aisec.cpg.graph.statements.expressions.MemberExpression import de.fraunhofer.aisec.cpg.graph.types.* import de.fraunhofer.aisec.cpg.helpers.SubgraphWalker.ScopedWalker import de.fraunhofer.aisec.cpg.helpers.Util +import de.fraunhofer.aisec.cpg.passes.inference.Inference.TypeInferenceObserver import de.fraunhofer.aisec.cpg.passes.inference.startInference import de.fraunhofer.aisec.cpg.passes.order.DependsOn import org.slf4j.LoggerFactory @@ -87,10 +88,11 @@ open class VariableUsageResolver(ctx: TranslationContext) : SymbolResolverPass(c } } - protected fun resolveFunctionPtr(reference: DeclaredReferenceExpression): ValueDeclaration? { - // Without FunctionPointerType, we cannot resolve function pointers - val fptrType = reference.type as? FunctionPointerType ?: return null - + /** This function seems to resolve function pointers pointing to a [MethodDeclaration]. */ + protected fun resolveMethodFunctionPointer( + reference: DeclaredReferenceExpression, + type: FunctionPointerType + ): ValueDeclaration { val parent = reference.name.parent return handleUnknownFunction( @@ -100,7 +102,7 @@ open class VariableUsageResolver(ctx: TranslationContext) : SymbolResolverPass(c null }, reference.name, - fptrType + type ) } @@ -144,8 +146,10 @@ open class VariableUsageResolver(ctx: TranslationContext) : SymbolResolverPass(c if (currentClass != null) { recordDeclType = currentClass.toType() } - if (current.type is FunctionPointerType && refersTo == null) { - refersTo = resolveFunctionPtr(current) + + val helperType = current.resolutionHelper?.type + if (helperType is FunctionPointerType && refersTo == null) { + refersTo = resolveMethodFunctionPointer(current, helperType) } // only add new nodes for non-static unknown @@ -246,9 +250,12 @@ open class VariableUsageResolver(ctx: TranslationContext) : SymbolResolverPass(c base.refersTo = baseTarget // Explicitly set the type of the call's base to the super type base.type = superType - // And set the possible subtypes, to ensure, that really only our + (base.refersTo as? HasType)?.type = superType + // And set the assigned subtypes, to ensure, that really only our // super type is in there - base.updatePossibleSubtypes(listOf(superType)) + base.assignedTypes = mutableSetOf(superType) + (base.refersTo as? ValueDeclaration)?.assignedTypes = + mutableSetOf(superType) } } } else { @@ -336,21 +343,24 @@ open class VariableUsageResolver(ctx: TranslationContext) : SymbolResolverPass(c .map { it.definition } .firstOrNull() } - // Attention: using orElse instead of orElseGet will always invoke unknown declaration - // handling! - return member ?: handleUnknownField(containingClass, reference.name, reference.type) + return member ?: handleUnknownField(containingClass, reference) } // TODO(oxisto): Move to inference class - protected fun handleUnknownField(base: Type, name: Name, type: Type): FieldDeclaration? { + protected fun handleUnknownField( + base: Type, + ref: DeclaredReferenceExpression + ): FieldDeclaration? { + val name = ref.name + // unwrap a potential pointer-type if (base is PointerType) { - return handleUnknownField(base.elementType, name, type) + return handleUnknownField(base.elementType, ref) } if (base.name !in recordMap) { // No matching record in the map? If we should infer it, we do so, otherwise we stop. - if (config?.inferenceConfiguration?.inferRecords != true) return null + if (!config.inferenceConfiguration.inferRecords) return null // We access an unknown field of an unknown record. so we need to handle that val kind = @@ -380,7 +390,8 @@ open class VariableUsageResolver(ctx: TranslationContext) : SymbolResolverPass(c val declaration = recordDeclaration.newFieldDeclaration( name.localName, - type, + // we will set the type later through the type inference observer + unknownType(), listOf(), "", null, @@ -389,6 +400,11 @@ open class VariableUsageResolver(ctx: TranslationContext) : SymbolResolverPass(c ) recordDeclaration.addField(declaration) declaration.isInferred = true + + // We might be able to resolve the type later (or better), if a type is + // assigned to our reference later + ref.registerTypeObserver(TypeInferenceObserver(declaration)) + declaration } } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/inference/Inference.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/inference/Inference.kt index 6d1af8f6f1..512a94f4e8 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/inference/Inference.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/inference/Inference.kt @@ -51,7 +51,6 @@ import org.slf4j.LoggerFactory */ class Inference(val start: Node, override val ctx: TranslationContext) : LanguageProvider, ScopeProvider, IsInferredProvider, ContextProvider { - val log: Logger = LoggerFactory.getLogger(Inference::class.java) override val language: Language<*>? get() = start.language @@ -92,7 +91,7 @@ class Inference(val start: Node, override val ctx: TranslationContext) : newFunctionDeclaration(name ?: "", code) } - log.debug( + Companion.log.debug( "Inferred a new {} declaration {} with parameter types {}", if (inferred is MethodDeclaration) "method" else "function", inferred.name, @@ -311,12 +310,12 @@ class Inference(val start: Node, override val ctx: TranslationContext) : kind: String = "class" ): RecordDeclaration? { if (type !is ObjectType) { - log.error( + Companion.log.error( "Trying to infer a record declaration of a non-object type. Not sure what to do? Should we change the type?" ) return null } - log.debug( + Companion.log.debug( "Encountered an unknown record type ${type.typeName} during a call. We are going to infer that record" ) @@ -338,7 +337,7 @@ class Inference(val start: Node, override val ctx: TranslationContext) : // delegate further operations to the scope manager. We also save the old scope so we can // restore it. return inferInScopeOf(start) { - log.debug( + Companion.log.debug( "Inferring a new namespace declaration {} {}", name, if (path != null) { @@ -359,6 +358,54 @@ class Inference(val start: Node, override val ctx: TranslationContext) : inferred } } + + /** + * This class implements a [HasType.TypeObserver] and uses the observed type to set the + * [ValueDeclaration.declaredType] of a [ValueDeclaration], based on the types we see. It can be + * registered on objects that are used to "start" an inference, for example a + * [MemberExpression], which infers a [FieldDeclaration]. Once the type of the member expression + * becomes known, we can use this information to set the type of the field. + * + * For now, this implementation uses the first type that we "see" and once the type of our + * [declaration] is known, we ignore further updates. In a future implementation, we could try + * to fine-tune this, e.g. by finding a common type (such as an interface) that is more + * probable, if multiple types are assigned. + */ + class TypeInferenceObserver(var declaration: ValueDeclaration) : HasType.TypeObserver { + override fun typeChanged(newType: Type, src: HasType) { + // Only set a new type, if it is unknown for now + if (declaration.type is UnknownType) { + declaration.type = newType + } else { + // TODO(oxisto): We could "refine" the type here based on further type + // observations + } + } + + override fun assignedTypeChanged(assignedTypes: Set, src: HasType) { + // Only set a new type, if it is unknown for now + if (declaration.type is UnknownType) { + // For now, just set it if there is only one type + if (assignedTypes.size == 1) { + val type = assignedTypes.single() + log.debug( + "Inferring type of declaration {} to be {}", + declaration.name, + type.name + ) + + declaration.type = type + } + } else { + // TODO(oxisto): We could "refine" the type here based on further type + // observations + } + } + } + + companion object { + val log: Logger = LoggerFactory.getLogger(Inference::class.java) + } } /** Provides information about the inference status of a node. */ diff --git a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/GraphExamples.kt b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/GraphExamples.kt index 33d664c905..adf48e0eb2 100644 --- a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/GraphExamples.kt +++ b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/GraphExamples.kt @@ -293,14 +293,14 @@ class GraphExamples { URI("conditional_expression.cpp"), Region(5, 16, 5, 17) ) - } assign literal(2, t("int")), + } assignAsExpr { literal(2, t("int")) }, ref("b") { location = PhysicalLocation( URI("conditional_expression.cpp"), Region(5, 23, 5, 24) ) - } assign literal(3, t("int")) + } assignAsExpr { literal(3, t("int")) } ) } ref("a") { diff --git a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/InferenceTest.kt b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/InferenceTest.kt index be5f5f8ada..caa7ffcc94 100644 --- a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/InferenceTest.kt +++ b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/InferenceTest.kt @@ -27,41 +27,46 @@ package de.fraunhofer.aisec.cpg.enhancements import de.fraunhofer.aisec.cpg.GraphExamples import de.fraunhofer.aisec.cpg.assertLocalName -import de.fraunhofer.aisec.cpg.graph.byNameOrNull +import de.fraunhofer.aisec.cpg.graph.* import de.fraunhofer.aisec.cpg.graph.declarations.RecordDeclaration -import de.fraunhofer.aisec.cpg.graph.get -import kotlin.test.Test -import kotlin.test.assertEquals -import kotlin.test.assertNotNull +import kotlin.test.* class InferenceTest { @Test fun testRecordInference() { - val tu = - GraphExamples.getInferenceRecord() - .components - .firstOrNull() - ?.translationUnits - ?.firstOrNull() + val result = GraphExamples.getInferenceRecord() + val tu = result.components.firstOrNull()?.translationUnits?.firstOrNull() assertNotNull(tu) - val record = tu.byNameOrNull("T") - assertNotNull(record) - assertLocalName("T", record) - assertEquals(true, record.isInferred) - assertEquals("struct", record.kind) + with(tu) { + val main = tu.functions["main"] - assertEquals(2, record.fields.size) + val valueRef = main.refs["value"] + assertNotNull(valueRef) + assertContains(valueRef.assignedTypes, primitiveType("int")) - val valueField = record.fields["value"] - assertNotNull(valueField) - assertLocalName("int", valueField.type) + val nextRef = main.refs["next"] + assertNotNull(nextRef) + assertContains(nextRef.assignedTypes, objectType("T").pointer()) - val nextField = record.fields["next"] - assertNotNull(nextField) - assertLocalName("T*", nextField.type) + val record = tu.records["T"] + assertNotNull(record) + assertLocalName("T", record) + assertEquals(true, record.isInferred) + assertEquals("struct", record.kind) + + assertEquals(2, record.fields.size) + + val valueField = record.fields["value"] + assertNotNull(valueField) + assertLocalName("int", valueField.type) + + val nextField = record.fields["next"] + assertNotNull(nextField) + assertLocalName("T*", nextField.type) + } } @Test diff --git a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/FluentTest.kt b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/FluentTest.kt index 470dbdde28..63bb5d4eec 100644 --- a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/FluentTest.kt +++ b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/FluentTest.kt @@ -61,7 +61,8 @@ class FluentTest { elseStmt { call("printf") { literal("else") } } } } - call("do") { call("some::func") } + declare { variable("some", t("SomeClass")) } + call("do") { call("some.func") } returnStmt { ref("a") + literal(2) } } @@ -69,7 +70,6 @@ class FluentTest { } } } - val tu = result.translationUnits.firstOrNull() // Let's assert that we did this correctly val main = result.functions["main"] @@ -138,18 +138,17 @@ class FluentTest { assertNotNull(lit1) assertEquals(1, lit1.value) - // Third line is th - // e CallExpression (containing another MemberCallExpression as argument) - val call = main[2] as? CallExpression + // Fourth line is the CallExpression (containing another MemberCallExpression as argument) + val call = main[3] as? CallExpression assertNotNull(call) assertLocalName("do", call) val mce = call.arguments[0] as? MemberCallExpression assertNotNull(mce) - assertFullName("some::func", mce) + assertFullName("UNKNOWN.func", mce) - // Fourth line is the ReturnStatement - val returnStatement = main[3] as? ReturnStatement + // Fifth line is the ReturnStatement + val returnStatement = main[4] as? ReturnStatement assertNotNull(returnStatement) assertNotNull(returnStatement.scope) @@ -171,7 +170,8 @@ class FluentTest { VariableUsageResolver(result.finalCtx).accept(result.components.first()) - // Now the reference should be resolved + // Now the reference should be resolved and the MCE name set assertRefersTo(ref, variable) + assertFullName("SomeClass::func", mce) } } diff --git a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/ShortcutsTest.kt b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/ShortcutsTest.kt index 1f5f0c1d02..49ba3c23a7 100644 --- a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/ShortcutsTest.kt +++ b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/ShortcutsTest.kt @@ -25,9 +25,7 @@ */ package de.fraunhofer.aisec.cpg.graph -import de.fraunhofer.aisec.cpg.GraphExamples -import de.fraunhofer.aisec.cpg.TranslationConfiguration -import de.fraunhofer.aisec.cpg.TranslationResult +import de.fraunhofer.aisec.cpg.* import de.fraunhofer.aisec.cpg.frontends.TestLanguage import de.fraunhofer.aisec.cpg.graph.declarations.FunctionDeclaration import de.fraunhofer.aisec.cpg.graph.declarations.MethodDeclaration @@ -36,19 +34,9 @@ import de.fraunhofer.aisec.cpg.graph.declarations.VariableDeclaration import de.fraunhofer.aisec.cpg.graph.statements.CompoundStatement import de.fraunhofer.aisec.cpg.graph.statements.DeclarationStatement import de.fraunhofer.aisec.cpg.graph.statements.IfStatement -import de.fraunhofer.aisec.cpg.graph.statements.expressions.BinaryOperator -import de.fraunhofer.aisec.cpg.graph.statements.expressions.CallExpression -import de.fraunhofer.aisec.cpg.graph.statements.expressions.ConstructExpression -import de.fraunhofer.aisec.cpg.graph.statements.expressions.DeclaredReferenceExpression -import de.fraunhofer.aisec.cpg.graph.statements.expressions.Literal -import de.fraunhofer.aisec.cpg.graph.statements.expressions.MemberCallExpression -import de.fraunhofer.aisec.cpg.graph.statements.expressions.MemberExpression -import de.fraunhofer.aisec.cpg.graph.statements.expressions.NewExpression +import de.fraunhofer.aisec.cpg.graph.statements.expressions.* import de.fraunhofer.aisec.cpg.passes.EdgeCachePass -import kotlin.test.Test -import kotlin.test.assertEquals -import kotlin.test.assertNotNull -import kotlin.test.assertTrue +import kotlin.test.* import org.junit.jupiter.api.BeforeAll import org.junit.jupiter.api.TestInstance @@ -159,15 +147,25 @@ class ShortcutsTest { fun testCallersOf() { val classDecl = shortcutClassResult.records["ShortcutClass"] assertNotNull(classDecl) - val print = classDecl.byNameOrNull("print") + val print = classDecl.methods["print"] assertNotNull(print) - val actual = shortcutClassResult.callersOf(print) - val expected = mutableListOf() - val main = classDecl.byNameOrNull("main") + val main = classDecl.functions["main"] assertNotNull(main) + + val scRefs = main.refs("sc") + scRefs.forEach { + assertNotNull(it) + assertLocalName("ShortcutClass", it.type) + } + + val printCall = main.calls["print"] + assertFullName("ShortcutClass.print", printCall) expected.add(main) + + val actual = shortcutClassResult.callersOf(print) + assertTrue(expected.containsAll(actual)) assertTrue(actual.containsAll(expected)) } @@ -193,28 +191,37 @@ class ShortcutsTest { val nestedThen = thenStatement.thenStatement as CompoundStatement expected.add(nestedThen) expected.add(nestedThen.statements[0]) - expected.add((nestedThen.statements[0] as BinaryOperator).lhs) - expected.add(((nestedThen.statements[0] as BinaryOperator).lhs as MemberExpression).base) - expected.add((nestedThen.statements[0] as BinaryOperator).rhs) + expected.add((nestedThen.statements[0] as AssignExpression).lhs.first()) + expected.add( + ((nestedThen.statements[0] as AssignExpression).lhs.first() as MemberExpression).base + ) + expected.add((nestedThen.statements[0] as AssignExpression).rhs.first()) val nestedElse = thenStatement.elseStatement as CompoundStatement expected.add(nestedElse) expected.add(nestedElse.statements[0]) - expected.add((nestedElse.statements[0] as BinaryOperator).lhs) - expected.add(((nestedElse.statements[0] as BinaryOperator).lhs as MemberExpression).base) - expected.add((nestedElse.statements[0] as BinaryOperator).rhs) + expected.add((nestedElse.statements[0] as AssignExpression).lhs.first()) + expected.add( + ((nestedElse.statements[0] as AssignExpression).lhs.first() as MemberExpression).base + ) + expected.add((nestedElse.statements[0] as AssignExpression).rhs.first()) ifStatement.elseStatement?.let { expected.add(it) } expected.add((ifStatement.elseStatement as CompoundStatement).statements[0]) expected.add( - ((ifStatement.elseStatement as CompoundStatement).statements[0] as BinaryOperator).lhs + ((ifStatement.elseStatement as CompoundStatement).statements[0] as AssignExpression) + .lhs + .first() ) expected.add( - (((ifStatement.elseStatement as CompoundStatement).statements[0] as BinaryOperator).lhs - as MemberExpression) + (((ifStatement.elseStatement as CompoundStatement).statements[0] as AssignExpression) + .lhs + .first() as MemberExpression) .base ) expected.add( - ((ifStatement.elseStatement as CompoundStatement).statements[0] as BinaryOperator).rhs + ((ifStatement.elseStatement as CompoundStatement).statements[0] as AssignExpression) + .rhs + .first() ) assertTrue(expected.containsAll(actual)) @@ -264,8 +271,9 @@ class ShortcutsTest { ((((magic2.body as CompoundStatement).statements[1] as IfStatement).elseStatement as CompoundStatement) .statements[0] - as BinaryOperator) + as AssignExpression) .lhs + .first() val paramPassed2 = aAssignment2.followPrevDFGEdgesUntilHit { it is Literal<*> } assertEquals(1, paramPassed2.fulfilled.size) @@ -279,8 +287,9 @@ class ShortcutsTest { ((((magic.body as CompoundStatement).statements[0] as IfStatement).elseStatement as CompoundStatement) .statements[0] - as BinaryOperator) + as AssignExpression) .lhs + .first() val paramPassed = attrAssignment.followPrevDFGEdgesUntilHit { it is Literal<*> } assertEquals(1, paramPassed.fulfilled.size) @@ -299,8 +308,9 @@ class ShortcutsTest { ((((magic.body as CompoundStatement).statements[0] as IfStatement).elseStatement as CompoundStatement) .statements[0] - as BinaryOperator) + as AssignExpression) .lhs + .first() val paramPassed = attrAssignment.followPrevEOGEdgesUntilHit { it is Literal<*> } assertEquals(1, paramPassed.fulfilled.size) @@ -324,9 +334,9 @@ class ShortcutsTest { val paramPassed = ifCondition.followNextEOGEdgesUntilHit { - it is BinaryOperator && + it is AssignExpression && it.operatorCode == "=" && - (it.rhs as? DeclaredReferenceExpression)?.refersTo == + (it.rhs.first() as? DeclaredReferenceExpression)?.refersTo == (ifCondition.lhs as DeclaredReferenceExpression).refersTo } assertEquals(1, paramPassed.fulfilled.size) @@ -344,8 +354,9 @@ class ShortcutsTest { ((((magic.body as CompoundStatement).statements[0] as IfStatement).elseStatement as CompoundStatement) .statements[0] - as BinaryOperator) + as AssignExpression) .lhs + .first() val paramPassed = attrAssignment.followPrevDFG { it is Literal<*> } assertNotNull(paramPassed) diff --git a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/AssignExpressionTest.kt b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/AssignExpressionTest.kt index 9d59e76ad2..a99c3f8647 100644 --- a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/AssignExpressionTest.kt +++ b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/AssignExpressionTest.kt @@ -25,7 +25,6 @@ */ package de.fraunhofer.aisec.cpg.graph.statements.expressions -import de.fraunhofer.aisec.cpg.assertLocalName import de.fraunhofer.aisec.cpg.frontends.TestLanguageFrontend import de.fraunhofer.aisec.cpg.graph.* import de.fraunhofer.aisec.cpg.graph.builder.function @@ -47,12 +46,12 @@ class AssignExpressionTest { val stmt = newAssignExpression(lhs = listOf(refA), rhs = listOf(refB)) // Type listeners should be configured - assertContains(refB.typeListeners, stmt) + assertContains(refB.typeObservers, stmt) - // Suddenly, we now we know the type of b. + // Suddenly, we now we know the type of "b". refB.type = objectType("MyClass") - // It should now propagate to a - assertLocalName("MyClass", refA.type) + // It should now propagate to the assigned type of "a" + assertContains(refA.assignedTypes, objectType("MyClass")) val assignments = stmt.assignments assertEquals(1, assignments.size) @@ -70,6 +69,7 @@ class AssignExpressionTest { "func", returnTypes = listOf(objectType("MyClass"), objectType("error")) ) + function("main") { val refA = newDeclaredReferenceExpression("a") val refErr = newDeclaredReferenceExpression("err") @@ -89,31 +89,36 @@ class AssignExpressionTest { } val tu = result.translationUnits.firstOrNull() - val call = tu.calls["func"] - val func = tu.functions["func"] - val refA = tu.refs["a"] - val refErr = tu.refs["err"] - - assertNotNull(call) - assertNotNull(func) - assertNotNull(refA) - assertNotNull(refErr) - - // This should now set the correct type of the call expression - call.invokes = listOf(func) - assertIs(call.type) - - assertLocalName("MyClass", refA.type) - assertLocalName("error", refErr.type) - - // Invoke the DFG pass - DFGPass(ctx).accept(result.components.first()) - - assertTrue(refA.prevDFG.contains(call)) - assertTrue(refErr.prevDFG.contains(call)) - - val assignments = tu.assignments - assertEquals(2, assignments.size) + with(tu) { + val call = tu.calls["func"] + val func = tu.functions["func"] + val refA = tu.refs["a"] + val refErr = tu.refs["err"] + + assertNotNull(call) + assertNotNull(func) + assertNotNull(refA) + assertNotNull(refErr) + + // This should now set the correct type of the call expression + call.invokes = listOf(func) + assertIs(call.type) + + // We should at least know the "assigned" type of the references. Their declared + // type is + // still unknown to us, because we don't know the declarations. + assertContains(refA.assignedTypes, objectType("MyClass")) + assertContains(refErr.assignedTypes, objectType("error")) + + // Invoke the DFG pass + DFGPass(ctx).accept(result.components.first()) + + assertTrue(refA.prevDFG.contains(call)) + assertTrue(refErr.prevDFG.contains(call)) + + val assignments = tu.assignments + assertEquals(2, assignments.size) + } } } } diff --git a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/types/TypePropagationTest.kt b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/types/TypePropagationTest.kt index 2faaebd0ab..22449001cd 100644 --- a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/types/TypePropagationTest.kt +++ b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/types/TypePropagationTest.kt @@ -25,14 +25,17 @@ */ package de.fraunhofer.aisec.cpg.graph.types +import de.fraunhofer.aisec.cpg.* import de.fraunhofer.aisec.cpg.frontends.TestLanguageFrontend import de.fraunhofer.aisec.cpg.graph.* import de.fraunhofer.aisec.cpg.graph.builder.* import de.fraunhofer.aisec.cpg.graph.declarations.VariableDeclaration import de.fraunhofer.aisec.cpg.graph.statements.CompoundStatement -import de.fraunhofer.aisec.cpg.graph.statements.DeclarationStatement -import de.fraunhofer.aisec.cpg.graph.statements.expressions.BinaryOperator +import de.fraunhofer.aisec.cpg.graph.statements.ReturnStatement +import de.fraunhofer.aisec.cpg.graph.statements.expressions.AssignExpression import de.fraunhofer.aisec.cpg.graph.statements.expressions.DeclaredReferenceExpression +import de.fraunhofer.aisec.cpg.passes.ControlFlowSensitiveDFGPass +import de.fraunhofer.aisec.cpg.passes.EvaluationOrderGraphPass import de.fraunhofer.aisec.cpg.passes.VariableUsageResolver import kotlin.test.* @@ -59,13 +62,22 @@ class TypePropagationTest { } } - val binaryOp = - (((result.functions["main"]?.body as? CompoundStatement)?.statements?.get(2) - as? DeclarationStatement) - ?.singleDeclaration as? VariableDeclaration) - ?.initializer + VariableUsageResolver(result.finalCtx).accept(result.components.first()) + + val intVar = result.variables["intVar"] + assertNotNull(intVar) + assertLocalName("int", intVar.type) + + val intVarRef = result.refs["intVar"] + assertNotNull(intVarRef) + assertLocalName("int", intVarRef.type) + val addResult = result.variables["addResult"] + assertNotNull(addResult) + + val binaryOp = addResult.initializer assertNotNull(binaryOp) + assertTrue(binaryOp.type is IntegerType) assertEquals("int", (binaryOp.type as IntegerType).name.toString()) assertEquals(32, (binaryOp.type as IntegerType).bitWidth) @@ -73,9 +85,24 @@ class TypePropagationTest { @Test fun testAssignTypePropagation() { - // TODO: This test is related to issue 1071 (it models case 2). + val frontend = TestLanguageFrontend() + + /** + * This roughly represents the following program in C: + * ```c + * int main() { + * int intVar; + * short shortVar; + * shortVar = intVar; + * return shortVar; + * } + * ``` + * + * `shortVar` and `intVar` should hold `short` and `int` as their respective [HasType.type]. + * The assignment will then propagate `int` as the [HasType.assignedTypes] to `shortVar`. + */ val result = - TestLanguageFrontend().build { + frontend.build { translationResult { translationUnit("test") { function("main", t("int")) { @@ -83,39 +110,312 @@ class TypePropagationTest { declare { variable("intVar", t("int")) {} } declare { variable("shortVar", t("short")) {} } ref("shortVar") assign ref("intVar") - returnStmt { literal(0) } + returnStmt { ref("shortVar") } } } } } } + VariableUsageResolver(result.finalCtx).accept(result.components.first()) + EvaluationOrderGraphPass(result.finalCtx) + .accept(result.components.first().translationUnits.first()) + ControlFlowSensitiveDFGPass(result.finalCtx) + .accept(result.components.first().translationUnits.first()) - val binaryOp = - (result.functions["main"]?.body as? CompoundStatement)?.statements?.get(2) - as? BinaryOperator - assertNotNull(binaryOp) + with(frontend) { + val main = result.functions["main"] + assertNotNull(main) - val rhs = binaryOp.rhs as? DeclaredReferenceExpression - assertNotNull(rhs) - assertTrue(rhs.type is IntegerType) - assertEquals("int", (rhs.type as IntegerType).name.toString()) - assertEquals(32, (rhs.type as IntegerType).bitWidth) + val assign = (main.body as? CompoundStatement)?.statements?.get(2) as? AssignExpression + assertNotNull(assign) - assertTrue(binaryOp.type is IntegerType) - assertEquals("short", (binaryOp.type as IntegerType).name.toString()) - assertEquals(16, (binaryOp.type as IntegerType).bitWidth) - - val lhs = binaryOp.lhs as? DeclaredReferenceExpression - assertNotNull(lhs) - assertTrue(lhs.type is IntegerType) - assertEquals("short", (lhs.type as IntegerType).name.toString()) - assertEquals(16, (lhs.type as IntegerType).bitWidth) - - val refersTo = lhs.refersTo as? VariableDeclaration - assertNotNull(refersTo) - assertTrue(refersTo.type is IntegerType) - assertEquals("short", (refersTo.type as IntegerType).name.toString()) - assertEquals(16, (refersTo.type as IntegerType).bitWidth) + val shortVar = main.variables["shortVar"] + assertNotNull(shortVar) + // At this point, shortVar should only have "short" as type and assigned types + assertEquals(primitiveType("short"), shortVar.type) + assertEquals(setOf(primitiveType("short")), shortVar.assignedTypes) + + val rhs = assign.rhs.firstOrNull() as? DeclaredReferenceExpression + assertNotNull(rhs) + assertIs(rhs.type) + assertLocalName("int", rhs.type) + assertEquals(32, (rhs.type as IntegerType).bitWidth) + + val shortVarRefLhs = assign.lhs.firstOrNull() as? DeclaredReferenceExpression + assertNotNull(shortVarRefLhs) + // At this point, shortVar was target of an assignment of an int variable, however, the + // int gets truncated into a short, so only short is part of the assigned types. + assertEquals(primitiveType("short"), shortVarRefLhs.type) + assertEquals(setOf(primitiveType("short")), shortVarRefLhs.assignedTypes) + + val shortVarRefReturnValue = + main.allChildren().firstOrNull()?.returnValue + assertNotNull(shortVarRefReturnValue) + // Finally, the assigned types should propagate along the DFG + assertEquals(setOf(primitiveType("short")), shortVarRefLhs.assignedTypes) + + val refersTo = shortVarRefLhs.refersTo as? VariableDeclaration + assertNotNull(refersTo) + assertIs(refersTo.type) + assertLocalName("short", refersTo.type) + assertEquals(16, (refersTo.type as IntegerType).bitWidth) + } + } + + @Test + fun testNewPropagation() { + val frontend = TestLanguageFrontend() + + /** + * This roughly represents the following C++ code: + * ```cpp + * int main() { + * BaseClass *b = new DerivedClass(); + * b.doSomething(); + * } + * ``` + */ + val result = + frontend.build { + translationResult { + translationUnit("test") { + function("main", t("int")) { + body { + declare { + variable("b", t("BaseClass").pointer()) { + new { + construct("DerivedClass") + type = t("DerivedClass").pointer() + } + } + } + call("b.doSomething") + } + } + } + } + } + + VariableUsageResolver(result.finalCtx).accept(result.components.first()) + + with(frontend) { + val main = result.functions["main"] + assertNotNull(main) + + val b = main.variables["b"] + assertNotNull(b) + assertEquals(objectType("BaseClass").pointer(), b.type) + assertEquals( + setOf( + objectType("BaseClass").pointer(), + objectType("DerivedClass").pointer(), + ), + b.assignedTypes + ) + + val bRef = main.refs["b"] + assertNotNull(bRef) + assertEquals(b.type, bRef.type) + assertEquals(b.assignedTypes, bRef.assignedTypes) + } + } + + @Test + fun testComplexPropagation() { + val frontend = + TestLanguageFrontend( + ctx = + TranslationContext( + TranslationConfiguration.builder().defaultPasses().build(), + ScopeManager(), + TypeManager() + ) + ) + + /** + * This is a more complex scenario in which we want to follow some of the data-flows and see + * how the types propagate. + * + * This roughly represents the following C++ code: + * ```cpp + * class BaseClass { + * virtual void doSomething(); + * }; + * class DerivedClassA : public BaseClass { + * void doSomething(); + * }; + * class DerivedClassB : public BaseClass { + * void doSomething(); + * }; + * + * BaseClass *create(bool flip) + * { + * // Create memory for a pointer to the base class + * BaseClass *b; + * // Create either DerivedClassA or DerivedClassB. This should assign both to the assigned types + * b = (flip == true) ? (BaseClass *)new DerivedClassA() : (BaseClass *)new DerivedClassB(); + * + * // Create a new array of pointers and assign our base class pointer to it + * auto bb = {b} + * + * // Return the first element again with an array subscription expression + * return bb[0]; + * } + * + * int main() { + * // Call the create function. We don't know which of the derived classes we return + * BaseClass *b = create(random); + * b->doSomething(); + * } + * ``` + */ + val result = + frontend.build { + translationResult { + translationUnit("test") { + record("BaseClass") { method("doSomething") } + record("DerivedClassA") { + superClasses = mutableListOf(t("BaseClass")) + method("doSomething") + } + record("DerivedClassB") { + superClasses = mutableListOf(t("BaseClass")) + method("doSomething") + } + function("create", t("BaseClass").pointer().pointer()) { + param("flip", t("bool")) + body { + declare { variable("b", t("BaseClass").pointer()) } + ref("b") assign + { + conditional( + ref("flip") eq literal(true), + cast(t("BaseClass").pointer()) { + new { + construct("DerivedClassA") + type = t("DerivedClassA").pointer() + } + }, + cast(t("BaseClass").pointer()) { + new { + construct("DerivedClassB") + type = t("DerivedClassB").pointer() + } + }, + ) + } + declare { + variable("bb", autoType()) { + ile(t("BaseClass").pointer().array()) { ref("b") } + } + } + returnStmt { + ase { + ref("bb") + literal(1) + } + } + } + } + function("main", t("int")) { + body { + declare { variable("random", t("bool")) } + declare { + variable("b", t("BaseClass").pointer()) { + call("create") { ref("random") } + } + } + call("b.doSomething") + } + } + } + } + } + + val baseClass = result.records["BaseClass"] + assertNotNull(baseClass) + + val derivedClassA = result.records["DerivedClassA"] + assertNotNull(derivedClassA) + assertContains(derivedClassA.superTypeDeclarations, baseClass) + + val derivedClassB = result.records["DerivedClassB"] + assertNotNull(derivedClassB) + assertContains(derivedClassB.superTypeDeclarations, baseClass) + + val create = result.functions["create"] + assertNotNull(create) + + with(create) { + val b = variables["b"] + assertNotNull(b) + assertEquals(objectType("BaseClass").pointer(), b.type) + + val bRefs = refs("b") + bRefs.forEach { + // The "type" of a reference must always be the same as its declaration + assertEquals(b.type, it.type) + // The assigned types should now contain both classes and the base class + assertEquals( + setOf( + objectType("BaseClass").pointer(), + objectType("DerivedClassA").pointer(), + objectType("DerivedClassB").pointer() + ), + it.assignedTypes + ) + } + + val assign = (body as CompoundStatement).statements(1) + assertNotNull(assign) + + val bb = variables["bb"] + assertNotNull(bb) + // Auto type based on the initializer's type + assertEquals(objectType("BaseClass").pointer().array(), bb.type) + // Assigned types should additionally contain our two derived classes + assertEquals( + setOf( + objectType("BaseClass").pointer().array(), + objectType("DerivedClassA").pointer().array(), + objectType("DerivedClassB").pointer().array() + ), + bb.assignedTypes + ) + + val returnStatement = (body as CompoundStatement).statements(3) + assertNotNull(returnStatement) + + val returnValue = returnStatement.returnValue + assertNotNull(returnValue) + assertEquals(objectType("BaseClass").pointer(), returnValue.type) + // The assigned types should now contain both classes and the base class in a non-array + // form, since we are using a single element of the array + assertEquals( + setOf( + objectType("BaseClass").pointer(), + objectType("DerivedClassA").pointer(), + objectType("DerivedClassB").pointer() + ), + returnValue.assignedTypes + ) + + // At this point we stop for now since we do not properly propagate the types across + // functions (yet) + } + + val main = result.functions["main"] + assertNotNull(main) + + with(main) { + val createCall = main.calls["create"] + assertNotNull(createCall) + assertContains(createCall.invokes, create) + + val b = main.variables["b"] + assertNotNull(b) + assertEquals(objectType("BaseClass").pointer(), b.type) + } } } diff --git a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/helpers/SubgraphWalkerTest.kt b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/helpers/SubgraphWalkerTest.kt index 3bbc46d397..cb8f7c3b37 100644 --- a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/helpers/SubgraphWalkerTest.kt +++ b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/helpers/SubgraphWalkerTest.kt @@ -63,7 +63,6 @@ internal class SubgraphWalkerTest : BaseTest() { .disableCleanup() .debugParser(true) .failOnError(true) - .typeSystemActiveInFrontend(false) .useParallelFrontends(true) .defaultLanguages() .registerLanguage(TestLanguage(".")) diff --git a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/passes/DFGTest.kt b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/passes/DFGTest.kt index ec7ff3098c..db88bc6721 100644 --- a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/passes/DFGTest.kt +++ b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/passes/DFGTest.kt @@ -125,7 +125,7 @@ class DFGTest { val result = GraphExamples.getDelayedAssignmentAfterRHS() val binaryOperatorAssignment = - TestUtils.findByUniqueName(result.allChildren(), "=") + TestUtils.findByUniqueName(result.allChildren(), "=") assertNotNull(binaryOperatorAssignment) val binaryOperatorAddition = @@ -138,7 +138,7 @@ class DFGTest { val varB = TestUtils.findByUniqueName(result.variables, "b") assertNotNull(varB) - val lhsA = binaryOperatorAssignment.lhs as DeclaredReferenceExpression + val lhsA = binaryOperatorAssignment.lhs.first() as DeclaredReferenceExpression val rhsA = binaryOperatorAddition.lhs as DeclaredReferenceExpression val b = TestUtils.findByUniqueName(result.refs, "b") assertNotNull(b) diff --git a/cpg-core/src/testFixtures/kotlin/de/fraunhofer/aisec/cpg/TestUtils.kt b/cpg-core/src/testFixtures/kotlin/de/fraunhofer/aisec/cpg/TestUtils.kt index d36e20e5b1..79435e8922 100644 --- a/cpg-core/src/testFixtures/kotlin/de/fraunhofer/aisec/cpg/TestUtils.kt +++ b/cpg-core/src/testFixtures/kotlin/de/fraunhofer/aisec/cpg/TestUtils.kt @@ -131,7 +131,6 @@ object TestUtils { .disableCleanup() .debugParser(true) .failOnError(true) - .typeSystemActiveInFrontend(false) .useParallelFrontends(true) .defaultLanguages() if (usePasses) { diff --git a/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CLanguage.kt b/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CLanguage.kt index c9960715ed..5d7f0cc4cb 100644 --- a/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CLanguage.kt +++ b/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CLanguage.kt @@ -93,8 +93,22 @@ open class CLanguage : "long double" to FloatingPointType("long double", 128, this, NumericType.Modifier.SIGNED), - // Convenience types, defined in + // Convenience types, defined in headers such as or . They are not + // part of the language per se, but part of the standard library. We therefore also + // consider them to be "built-in" types, because we often don't parse all the headers + // which define them internally. "bool" to IntegerType("bool", 1, this, NumericType.Modifier.SIGNED), + "int8_t" to IntegerType("int8_t", 8, this, NumericType.Modifier.SIGNED), + "int16_t" to IntegerType("int16_t", 16, this, NumericType.Modifier.SIGNED), + "int32_t" to IntegerType("int32_t", 32, this, NumericType.Modifier.SIGNED), + "int64_t" to IntegerType("int64_t", 64, this, NumericType.Modifier.SIGNED), + "uint8_t" to IntegerType("uint8_t", 8, this, NumericType.Modifier.UNSIGNED), + "uint16_t" to IntegerType("uint16_t", 16, this, NumericType.Modifier.UNSIGNED), + "uint32_t" to IntegerType("uint32_t", 32, this, NumericType.Modifier.UNSIGNED), + "uint64_t" to IntegerType("uint64_t", 64, this, NumericType.Modifier.UNSIGNED), + + // Other commonly used extension types + "__int128" to IntegerType("__int128", 128, this, NumericType.Modifier.SIGNED), ) override fun refineNormalCallResolution( @@ -139,6 +153,6 @@ open class CLanguage : listOf( *ctx.scopeManager.resolveFunctionStopScopeTraversalOnDefinition(call).toTypedArray() ) - return resolveWithImplicitCast(call, initialInvocationCandidates, ctx) + return resolveWithImplicitCast(call, initialInvocationCandidates) } } diff --git a/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CPPLanguage.kt b/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CPPLanguage.kt index 8c22732914..03ade6b6a3 100644 --- a/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CPPLanguage.kt +++ b/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CPPLanguage.kt @@ -78,6 +78,7 @@ class CPPLanguage : "signed char" to IntegerType("signed char", 8, this, NumericType.Modifier.SIGNED), "unsigned char" to IntegerType("unsigned char", 8, this, NumericType.Modifier.UNSIGNED), "char" to IntegerType("char", 8, this, NumericType.Modifier.NOT_APPLICABLE), + "wchar_t" to IntegerType("wchar_t", 32, this, NumericType.Modifier.NOT_APPLICABLE), "char8_t" to IntegerType("char8_t", 8, this, NumericType.Modifier.NOT_APPLICABLE), "char16_t" to IntegerType("char16_t", 16, this, NumericType.Modifier.NOT_APPLICABLE), "char32_t" to IntegerType("char32_t", 32, this, NumericType.Modifier.NOT_APPLICABLE), @@ -88,8 +89,21 @@ class CPPLanguage : "long double" to FloatingPointType("long double", 128, this, NumericType.Modifier.SIGNED), - // Some convenience types + // Convenience types, defined in headers. They are not part of the language per se, but + // part of the standard library. We therefore also consider them to be "built-in" types, + // because we often don't parse all the headers which define them internally. "std::string" to StringType("std::string", this), + "int8_t" to IntegerType("int8_t", 8, this, NumericType.Modifier.SIGNED), + "int16_t" to IntegerType("int16_t", 16, this, NumericType.Modifier.SIGNED), + "int32_t" to IntegerType("int32_t", 32, this, NumericType.Modifier.SIGNED), + "int64_t" to IntegerType("int64_t", 64, this, NumericType.Modifier.SIGNED), + "uint8_t" to IntegerType("uint8_t", 8, this, NumericType.Modifier.UNSIGNED), + "uint16_t" to IntegerType("uint16_t", 16, this, NumericType.Modifier.UNSIGNED), + "uint32_t" to IntegerType("uint32_t", 32, this, NumericType.Modifier.UNSIGNED), + "uint64_t" to IntegerType("uint64_t", 64, this, NumericType.Modifier.UNSIGNED), + + // Other commonly used extension types + "__int128" to IntegerType("__int128", 128, this, NumericType.Modifier.SIGNED), ) /** @@ -174,8 +188,7 @@ class CPPLanguage : call, recordDeclaration.methods.filter { m -> namePattern.matcher(m.name).matches() /*&& !m.isImplicit()*/ - }, - ctx + } ) ) } diff --git a/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CXXHandler.kt b/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CXXHandler.kt index 3b2cba1412..abe28bed70 100644 --- a/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CXXHandler.kt +++ b/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CXXHandler.kt @@ -37,6 +37,7 @@ abstract class CXXHandler( configConstructor: Supplier, lang: CXXLanguageFrontend ) : Handler(configConstructor, lang) { + /** * We intentionally override the logic of [Handler.handle] because we do not want the map-based * logic, but rather want to make use of the Kotlin-when syntax. @@ -65,6 +66,8 @@ abstract class CXXHandler( frontend.setComment(node, ctx) frontend.process(ctx, node) + this.lastNode = node + return node } diff --git a/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CXXLanguageFrontend.kt b/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CXXLanguageFrontend.kt index 0e9d379fe9..69276887ca 100644 --- a/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CXXLanguageFrontend.kt +++ b/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CXXLanguageFrontend.kt @@ -32,10 +32,7 @@ import de.fraunhofer.aisec.cpg.frontends.LanguageFrontend import de.fraunhofer.aisec.cpg.frontends.TranslationException import de.fraunhofer.aisec.cpg.graph.* import de.fraunhofer.aisec.cpg.graph.Annotation -import de.fraunhofer.aisec.cpg.graph.declarations.ConstructorDeclaration -import de.fraunhofer.aisec.cpg.graph.declarations.Declaration -import de.fraunhofer.aisec.cpg.graph.declarations.FunctionDeclaration -import de.fraunhofer.aisec.cpg.graph.declarations.TranslationUnitDeclaration +import de.fraunhofer.aisec.cpg.graph.declarations.* import de.fraunhofer.aisec.cpg.graph.statements.expressions.Expression import de.fraunhofer.aisec.cpg.graph.types.* import de.fraunhofer.aisec.cpg.helpers.Benchmark @@ -206,6 +203,7 @@ class CXXLanguageFrontend(language: Language, ctx: Translat val symbols: HashMap = HashMap() symbols.putAll(config.symbols) + includePaths.addAll(config.includePaths.map { it.toAbsolutePath().toString() }) config.compilationDatabase?.getIncludePaths(file)?.let { includePaths.addAll(it) } @@ -556,20 +554,33 @@ class CXXLanguageFrontend(language: Language, ctx: Translat val name = specifier.rawSignature return when { - // auto type; we model this as an unknown type. Maybe in the future, we will - // differentiate between unknown types and "auto-deduced" types + // auto type specifier.type == IASTSimpleDeclSpecifier.t_auto -> { - unknownType() + autoType() } // void type specifier.type == IASTSimpleDeclSpecifier.t_void -> { IncompleteType() } + // __typeof__ type + specifier.type == IASTSimpleDeclSpecifier.t_typeof -> { + objectType("typeof(${specifier.declTypeExpression.rawSignature})") + } + // A decl type + specifier.type == IASTSimpleDeclSpecifier.t_decltype -> { + objectType("decltype(${specifier.declTypeExpression.rawSignature})") + } // The type of constructor declaration is always the declaration itself specifier.type == IASTSimpleDeclSpecifier.t_unspecified && hint is ConstructorDeclaration -> { hint.name.parent?.let { objectType(it) } ?: unknownType() } + // The type of conversion operator is also always the declaration itself + specifier.type == IASTSimpleDeclSpecifier.t_unspecified && + hint is MethodDeclaration && + hint.name.localName == "operator#0" -> { + hint.name.parent?.let { objectType(it) } ?: unknownType() + } // C (not C++) allows unspecified types in function declarations, they // default to int and usually produce a warning name == "" && language !is CPPLanguage -> { @@ -595,7 +606,18 @@ class CXXLanguageFrontend(language: Language, ctx: Translat // We need to remove qualifiers such as "const" from the name here, because // we model them as part of the variable declaration and not the type, so use // the "canonical" name - primitiveType(specifier.canonicalName) + val canonicalName = specifier.canonicalName + if (canonicalName == "") { + Util.errorWithFileLocation( + this, + specifier, + log, + "Could not determine canonical name for potential primitive type $name" + ) + newProblemType() + } else { + primitiveType(canonicalName) + } } } } @@ -777,14 +799,17 @@ private val IASTSimpleDeclSpecifier.canonicalName: CharSequence // Last part is the actual type (int, float, ...) when (type) { IASTSimpleDeclSpecifier.t_char -> parts += "char" + IASTSimpleDeclSpecifier.t_wchar_t -> parts += "wchar_t" IASTSimpleDeclSpecifier.t_char16_t -> parts += "char16_t" - IASTSimpleDeclSpecifier.t_char32_t -> parts += "chat32_t" + IASTSimpleDeclSpecifier.t_char32_t -> parts += "char32_t" IASTSimpleDeclSpecifier.t_int -> parts += "int" IASTSimpleDeclSpecifier.t_float -> parts += "float" IASTSimpleDeclSpecifier.t_double -> parts += "double" IASTSimpleDeclSpecifier.t_bool -> parts += "bool" IASTSimpleDeclSpecifier.t_unspecified -> { - // nothing to do + if (isSigned || isUnsigned) { + parts += "int" + } } IASTSimpleDeclSpecifier.t_auto -> parts = mutableListOf("auto") else -> { diff --git a/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/DeclarationHandler.kt b/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/DeclarationHandler.kt index 88a1faafe9..a70d6c933f 100644 --- a/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/DeclarationHandler.kt +++ b/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/DeclarationHandler.kt @@ -32,6 +32,8 @@ import de.fraunhofer.aisec.cpg.graph.scopes.TemplateScope import de.fraunhofer.aisec.cpg.graph.statements.CompoundStatement import de.fraunhofer.aisec.cpg.graph.statements.ReturnStatement import de.fraunhofer.aisec.cpg.graph.statements.Statement +import de.fraunhofer.aisec.cpg.graph.statements.expressions.DeclaredReferenceExpression +import de.fraunhofer.aisec.cpg.graph.statements.expressions.UnaryOperator import de.fraunhofer.aisec.cpg.graph.types.* import java.util.function.Supplier import org.eclipse.cdt.core.dom.ast.* @@ -410,14 +412,113 @@ class DeclarationHandler(lang: CXXLanguageFrontend) : } private fun handleSimpleDeclaration(ctx: IASTSimpleDeclaration): Declaration { - var primaryDeclaration: Declaration? = null val sequence = DeclarationSequence() + val declSpecifier = ctx.declSpecifier + + // check, whether the declaration specifier also contains declarations, e.g. class + // definitions or enums + val (primaryDeclaration, useNameOfDeclarator) = + handleDeclarationSpecifier(declSpecifier, ctx, sequence) + + // Fill template params, if needed + val templateParams: List? = extractTemplateParams(ctx, declSpecifier) + + // Loop through all declarators, as we can potentially have multiple declarations here + for (declarator in ctx.declarators) { + // If a previous step informed us that we should take the name of the primary + // declaration, we do so here. This most likely is the case of a typedef struct. + val declSpecifierToUse = + if (useNameOfDeclarator && declSpecifier is IASTCompositeTypeSpecifier) { + val copy = declSpecifier.copy() + copy.name = CPPASTName(primaryDeclaration?.name?.toString()?.toCharArray()) + copy + } else { + declSpecifier + } + + var type: Type + + if (ctx.isTypedef) { + type = frontend.typeOf(declarator, declSpecifierToUse) + + // Handle typedefs. + val declaration = handleTypedef(declarator, ctx, type) + + sequence.addDeclaration(declaration) + } else { + // Parse the declaration first, so we can supply the declaration as a hint to + // the typeOf function. + val declaration = + frontend.declaratorHandler.handle(declarator) as? ValueDeclaration ?: continue + + // Parse the type (with some hints) + type = frontend.typeOf(declarator, declSpecifierToUse, declaration) + + // For function *declarations*, we need to update the return types based on the + // function type. For function *definitions*, this is done in + // [handleFunctionDefinition]. + if (declaration is FunctionDeclaration) { + declaration.returnTypes = + (type as? FunctionType)?.returnTypes ?: listOf(IncompleteType()) + } + + // We also need to set the type, based on the declarator type. + declaration.type = type + + // process attributes + frontend.processAttributes(declaration, ctx) + sequence.addDeclaration(declaration) + + // We want to make sure that we parse the initializer *after* we have set the + // type. This has several advantages: + // * This way we can deduce, whether our initializer needs to have the + // declared type (in case of a ConstructExpression); + // * or if the declaration needs to have the same type as the initializer (when + // an auto-type is used). The latter case is done internally by the + // VariableDeclaration class and its type observer. + // * Additionally, it makes sure that the type is known before parsing the + // initializer. This allows us to guess cast vs. call expression in the + // initializer. + if (declaration is VariableDeclaration) { + // Set template parameters of the variable (if any) + declaration.templateParameters = templateParams + + // Parse the initializer, if we have one + declarator.initializer?.let { + val initializer = frontend.initializerHandler.handle(it) + when { + // We need to set a resolution "helper" for function pointers, so that a + // reference to this declaration can resolve the function pointer (using + // the type of this declaration). The typical (and only) scenario we + // support here is the assignment of a `&ref` as initializer. + initializer is UnaryOperator && type is FunctionPointerType -> { + val ref = initializer.input as? DeclaredReferenceExpression + ref?.resolutionHelper = declaration + } + } + + declaration.initializer = initializer + } + } + } + } + + return simplifySequence(sequence) + } + /** + * In C++, a [IASTDeclSpecifier] can potentially also contain declarations, e.g. records or + * enums. This function gathers these [Declaration] nodes, before processing the remainder of + * declarations within [IASTSimpleDeclaration.getDeclarators]. + */ + private fun handleDeclarationSpecifier( + declSpecifier: IASTDeclSpecifier?, + ctx: IASTSimpleDeclaration, + sequence: DeclarationSequence + ): Pair { + var primaryDeclaration: Declaration? = null var useNameOfDeclarator = false - // check, whether the declaration specifier also contains declarations, i.e. class - // definitions - val declSpecifier = ctx.declSpecifier when (declSpecifier) { is IASTCompositeTypeSpecifier -> { primaryDeclaration = @@ -444,6 +545,7 @@ class DeclarationHandler(lang: CXXLanguageFrontend) : } } frontend.processAttributes(primaryDeclaration, ctx) + sequence.addDeclaration(primaryDeclaration) } } @@ -463,72 +565,43 @@ class DeclarationHandler(lang: CXXLanguageFrontend) : } } + return Pair(primaryDeclaration, useNameOfDeclarator) + } + + /** + * Extracts template parameters (used for [VariableDeclaration.templateParameters] out of the + * declaration (if it has any), otherwise null is returned. + */ + private fun extractTemplateParams( + ctx: IASTSimpleDeclaration, + declSpecifier: IASTDeclSpecifier, + ): MutableList? { if ( !ctx.isTypedef && declSpecifier is CPPASTNamedTypeSpecifier && declSpecifier.name is CPPASTTemplateId ) { - handleTemplateUsage(declSpecifier, ctx, sequence) - } else { - for (declarator in ctx.declarators) { - // If a previous step informed us that we should take the name of the primary - // declaration, we do so here. This most likely is the case of a typedef struct. - val declSpecifierToUse = - if (useNameOfDeclarator && declSpecifier is IASTCompositeTypeSpecifier) { - val copy = declSpecifier.copy() - copy.name = CPPASTName(primaryDeclaration?.name?.toString()?.toCharArray()) - copy - } else { - declSpecifier - } - - var type: Type - - // If this is a variable declaration with initializer, it is important, that we - // parse the type first, so that the type is known before parsing the declaration. - // This allows us to guess cast vs. call expression in the initializer. - if (declarator !is IASTFunctionDeclarator && declarator.initializer != null) { - // We only need to parse it, but we are not really storing the result. That is - // not ideal, but probably the "best" way. - frontend.typeOf(declarator, declSpecifierToUse) - } - - if (ctx.isTypedef) { - type = frontend.typeOf(declarator, declSpecifierToUse) - - // Handle typedefs. - val declaration = handleTypedef(declarator, ctx, type) - - sequence.addDeclaration(declaration) - } else { - // Parse the declaration first, so we can supply the declaration as a hint to - // the typeOf function. - val declaration = - frontend.declaratorHandler.handle(declarator) as? ValueDeclaration - - type = frontend.typeOf(declarator, declSpecifierToUse, declaration) - - // For function *declarations*, we need to update the return types based on the - // function type. For function *definitions*, this is done in - // [handleFunctionDefinition]. - if (declaration is FunctionDeclaration) { - declaration.returnTypes = - (type as? FunctionType)?.returnTypes ?: listOf(IncompleteType()) - } - - if (declaration != null) { - // We also need to set the return type, based on the declarator type. - declaration.type = type - - // process attributes - frontend.processAttributes(declaration, ctx) - sequence.addDeclaration(declaration) - } + val templateParams = mutableListOf() + val templateId = declSpecifier.name as CPPASTTemplateId + for (templateArgument in templateId.templateArguments) { + if (templateArgument is CPPASTTypeId) { + val genericInstantiation = frontend.typeOf(templateArgument) + templateParams.add( + newTypeExpression( + genericInstantiation.name.toString(), + genericInstantiation, + ) + ) + } else if (templateArgument is IASTExpression) { + val expression = frontend.expressionHandler.handle(templateArgument) + expression?.let { templateParams.add(it) } } } + + return templateParams } - return simplifySequence(sequence) + return null } private fun handleTypedef( @@ -601,53 +674,6 @@ class DeclarationHandler(lang: CXXLanguageFrontend) : } } - /** - * Handles usage of Templates in SimpleDeclarations - * - * @param typeSpecifier - * @param ctx - * @param sequence - */ - private fun handleTemplateUsage( - typeSpecifier: CPPASTNamedTypeSpecifier, - ctx: IASTSimpleDeclaration, - sequence: DeclarationSequence - ) { - val templateId = typeSpecifier.name as CPPASTTemplateId - val templateParams = mutableListOf() - - for (templateArgument in templateId.templateArguments) { - if (templateArgument is CPPASTTypeId) { - val genericInstantiation = frontend.typeOf(templateArgument) - templateParams.add( - newTypeExpression( - genericInstantiation.name.toString(), - genericInstantiation, - ) - ) - } else if (templateArgument is IASTExpression) { - val expression = frontend.expressionHandler.handle(templateArgument) - expression?.let { templateParams.add(it) } - } - } - - for (declarator in ctx.declarators) { - val declaration = frontend.declaratorHandler.handle(declarator) as ValueDeclaration - - // Update Type - declaration.type = frontend.typeOf(declarator, typeSpecifier) - - // Set TemplateParameters into VariableDeclaration - if (declaration is VariableDeclaration) { - declaration.templateParameters = templateParams - } - - // process attributes - frontend.processAttributes(declaration, ctx) - sequence.addDeclaration(declaration) - } - } - private fun parseInclusions( includes: Array?, allIncludes: HashMap> diff --git a/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/DeclaratorHandler.kt b/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/DeclaratorHandler.kt index 7ed4122fb5..af98a6d8eb 100644 --- a/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/DeclaratorHandler.kt +++ b/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/DeclaratorHandler.kt @@ -35,7 +35,6 @@ import de.fraunhofer.aisec.cpg.graph.types.* import de.fraunhofer.aisec.cpg.helpers.Util import java.util.* import java.util.function.Supplier -import java.util.regex.Pattern import org.eclipse.cdt.core.dom.ast.* import org.eclipse.cdt.core.dom.ast.cpp.ICPPASTCompositeTypeSpecifier import org.eclipse.cdt.core.dom.ast.gnu.cpp.GPPLanguage @@ -122,12 +121,6 @@ class DeclaratorHandler(lang: CXXLanguageFrontend) : implicitInitializerAllowed, ) - // Parse the initializer, if we have one - val init = ctx.initializer - if (init != null) { - declaration.initializer = frontend.initializerHandler.handle(init) - } - // Add this declaration to the current scope frontend.scopeManager.addDeclaration(declaration) @@ -402,9 +395,6 @@ class DeclaratorHandler(lang: CXXLanguageFrontend) : } private fun handleFunctionPointer(ctx: IASTFunctionDeclarator, name: String): ValueDeclaration { - val initializer = - if (ctx.initializer == null) null - else frontend.initializerHandler.handle(ctx.initializer) // unfortunately we are not told whether this is a field or not, so we have to find it out // ourselves val result: ValueDeclaration @@ -412,25 +402,18 @@ class DeclaratorHandler(lang: CXXLanguageFrontend) : if (recordDeclaration == null) { // variable result = newVariableDeclaration(name, unknownType(), ctx.rawSignature, true) - result.initializer = initializer } else { // field val code = ctx.rawSignature - val namePattern = Pattern.compile("\\((\\*|.+\\*)(?[^)]*)") - val matcher = namePattern.matcher(code) - var fieldName: String? = "" - if (matcher.find()) { - fieldName = matcher.group("name").trim() - } result = newFieldDeclaration( - fieldName, + name, unknownType(), emptyList(), code, frontend.locationOf(ctx), - initializer, - true + null, + false, ) } diff --git a/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/ExpressionHandler.kt b/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/ExpressionHandler.kt index a4b7597780..9e4e4e7bc4 100644 --- a/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/ExpressionHandler.kt +++ b/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/ExpressionHandler.kt @@ -28,8 +28,6 @@ package de.fraunhofer.aisec.cpg.frontends.cxx import de.fraunhofer.aisec.cpg.graph.* import de.fraunhofer.aisec.cpg.graph.declarations.FunctionDeclaration import de.fraunhofer.aisec.cpg.graph.declarations.MethodDeclaration -import de.fraunhofer.aisec.cpg.graph.edge.Properties -import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge import de.fraunhofer.aisec.cpg.graph.statements.expressions.* import de.fraunhofer.aisec.cpg.graph.types.* import de.fraunhofer.aisec.cpg.helpers.Util @@ -44,6 +42,7 @@ import org.eclipse.cdt.core.dom.ast.IASTLiteralExpression.* import org.eclipse.cdt.core.dom.ast.cpp.ICPPASTLambdaExpression import org.eclipse.cdt.internal.core.dom.parser.c.CASTDesignatedInitializer import org.eclipse.cdt.internal.core.dom.parser.cpp.* +import org.eclipse.cdt.internal.core.model.ASTStringUtil /** * Note: CDT expresses hierarchies in Interfaces to allow to have multi-inheritance in java. Because @@ -64,8 +63,9 @@ class ExpressionHandler(lang: CXXLanguageFrontend) : is IASTFieldReference -> handleFieldReference(node) is IASTFunctionCallExpression -> handleFunctionCallExpression(node) is IASTCastExpression -> handleCastExpression(node) - is IASTInitializerList -> handleInitializerList(node) is IASTExpressionList -> handleExpressionList(node) + is IASTInitializerList -> frontend.initializerHandler.handle(node) + ?: ProblemExpression("could not parse initializer list") is IASTArraySubscriptExpression -> handleArraySubscriptExpression(node) is IASTTypeIdExpression -> handleTypeIdExpression(node) is CPPASTNewExpression -> handleNewExpression(node) @@ -103,7 +103,7 @@ class ExpressionHandler(lang: CXXLanguageFrontend) : } } - // By default, the outer variables passed by value to the lambda are unmutable. But we can + // By default, the outer variables passed by value to the lambda are immutable. But we can // either make the function "mutable" or pass everything by reference. lambda.areVariablesMutable = (node.declarator as? CPPASTFunctionDeclarator)?.isMutable == true || @@ -112,6 +112,7 @@ class ExpressionHandler(lang: CXXLanguageFrontend) : val anonymousFunction = node.declarator?.let { frontend.declaratorHandler.handle(it) as? FunctionDeclaration } ?: newFunctionDeclaration("lambda${lambda.hashCode()}") + anonymousFunction.type = FunctionType.computeType(anonymousFunction) frontend.scopeManager.enterScope(anonymousFunction) anonymousFunction.body = frontend.statementHandler.handle(node.body) @@ -211,6 +212,7 @@ class ExpressionHandler(lang: CXXLanguageFrontend) : // an implicit one initializer = newConstructExpression(t.name.localName, "${t.name.localName}()") initializer.isImplicit = true + initializer.type = t } // we also need to "forward" our template parameters (if we have any) to the construct @@ -290,8 +292,6 @@ class ExpressionHandler(lang: CXXLanguageFrontend) : if (isPrimitive(castExpression.castType) || ctx.operator == 4) { castExpression.type = castExpression.castType - } else { - castExpression.expression.registerTypeListener(castExpression) } return castExpression @@ -437,8 +437,8 @@ class ExpressionHandler(lang: CXXLanguageFrontend) : reference is CastExpression -> { // this really is a cast expression in disguise reference.expression = - handle(ctx.arguments.first()) - ?: ProblemExpression("could not parse argument for cast") + ctx.arguments.firstOrNull()?.let { handle(it) } + ?: newProblemExpression("could not parse argument for cast") return reference } else -> { @@ -476,46 +476,27 @@ class ExpressionHandler(lang: CXXLanguageFrontend) : return expressionList } - private fun handleBinaryExpression(ctx: IASTBinaryExpression): BinaryOperator { - var operatorCode = "" - when (ctx.operator) { - op_multiply -> operatorCode = "*" - op_divide -> operatorCode = "/" - op_modulo -> operatorCode = "%" - op_plus -> operatorCode = "+" - op_minus -> operatorCode = "-" - op_shiftLeft -> operatorCode = "<<" - op_shiftRight -> operatorCode = ">>" - op_lessThan -> operatorCode = "<" - op_greaterThan -> operatorCode = ">" - op_lessEqual -> operatorCode = "<=" - op_greaterEqual -> operatorCode = ">=" - op_binaryAnd -> operatorCode = "&" - op_binaryXor -> operatorCode = "^" - op_binaryOr -> operatorCode = "|" - op_logicalAnd -> operatorCode = "&&" - op_logicalOr -> operatorCode = "||" - op_assign -> operatorCode = "=" - op_multiplyAssign -> operatorCode = "*=" - op_divideAssign -> operatorCode = "/=" - op_moduloAssign -> operatorCode = "%=" - op_plusAssign -> operatorCode = "+=" - op_minusAssign -> operatorCode = "-=" - op_shiftLeftAssign -> operatorCode = "<<=" - op_shiftRightAssign -> operatorCode = ">>=" - op_binaryAndAssign -> operatorCode = "&=" - op_binaryXorAssign -> operatorCode = "^=" - op_binaryOrAssign -> operatorCode = "|=" - op_equals -> operatorCode = "==" - op_notequals -> operatorCode = "!=" - op_pmdot -> operatorCode = ".*" - op_pmarrow -> operatorCode = "->*" - op_max -> operatorCode = ">?" - op_min -> operatorCode = "?<" - op_ellipses -> operatorCode = "..." - else -> - Util.errorWithFileLocation(frontend, ctx, log, "unknown operator {}", ctx.operator) - } + private fun handleBinaryExpression(ctx: IASTBinaryExpression): Expression { + val operatorCode = + when (ctx.operator) { + op_assign, + op_multiplyAssign, + op_divideAssign, + op_moduloAssign, + op_plusAssign, + op_minusAssign, + op_shiftLeftAssign, + op_shiftRightAssign, + op_binaryAndAssign, + op_binaryXorAssign, + op_binaryOrAssign -> { + return handleAssignment(ctx) + } + op_pmdot -> ".*" + op_pmarrow -> "->*" + else -> String(ASTStringUtil.getBinaryOperatorString(ctx)) + } + val binaryOperator = newBinaryOperator(operatorCode, ctx.rawSignature) val lhs = handle(ctx.operand1) ?: newProblemExpression("could not parse lhs") val rhs = @@ -531,6 +512,25 @@ class ExpressionHandler(lang: CXXLanguageFrontend) : return binaryOperator } + private fun handleAssignment(ctx: IASTBinaryExpression): Expression { + val lhs = handle(ctx.operand1) ?: newProblemExpression("missing LHS") + val rhs = + if (ctx.operand2 != null) { + handle(ctx.operand2) + } else { + handle(ctx.initOperand2) + } + ?: newProblemExpression("missing RHS") + + val operatorCode = String(ASTStringUtil.getBinaryOperatorString(ctx)) + val assign = newAssignExpression(operatorCode, listOf(lhs), listOf(rhs), rawNode = ctx) + if (rhs is UnaryOperator && rhs.input is DeclaredReferenceExpression) { + (rhs.input as DeclaredReferenceExpression).resolutionHelper = lhs + } + + return assign + } + private fun handleLiteralExpression(ctx: IASTLiteralExpression): Expression { return when (ctx.kind) { lk_integer_constant -> handleIntegerLiteral(ctx) @@ -550,22 +550,6 @@ class ExpressionHandler(lang: CXXLanguageFrontend) : } } - private fun handleInitializerList(ctx: IASTInitializerList): InitializerListExpression { - val expression = newInitializerListExpression(ctx.rawSignature) - - for (clause in ctx.clauses) { - handle(clause)?.let { - val edge = PropertyEdge(expression, it) - edge.addProperty(Properties.INDEX, expression.initializerEdges.size) - - expression.initializerEdges.add(edge) - expression.addPrevDFG(it) - } - } - - return expression - } - private fun handleCXXDesignatedInitializer( ctx: CPPASTDesignatedInitializer ): DesignatedInitializerExpression { diff --git a/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/InitializerHandler.kt b/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/InitializerHandler.kt index be3311744a..dd7be1da52 100644 --- a/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/InitializerHandler.kt +++ b/cpg-language-cxx/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/InitializerHandler.kt @@ -25,10 +25,17 @@ */ package de.fraunhofer.aisec.cpg.frontends.cxx +import de.fraunhofer.aisec.cpg.graph.declarations.ValueDeclaration +import de.fraunhofer.aisec.cpg.graph.declarations.VariableDeclaration +import de.fraunhofer.aisec.cpg.graph.edge.Properties +import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge import de.fraunhofer.aisec.cpg.graph.newConstructExpression +import de.fraunhofer.aisec.cpg.graph.newInitializerListExpression import de.fraunhofer.aisec.cpg.graph.newProblemExpression import de.fraunhofer.aisec.cpg.graph.statements.expressions.Expression +import de.fraunhofer.aisec.cpg.graph.statements.expressions.InitializerListExpression import de.fraunhofer.aisec.cpg.graph.statements.expressions.ProblemExpression +import de.fraunhofer.aisec.cpg.graph.unknownType import java.util.function.Supplier import org.eclipse.cdt.core.dom.ast.IASTEqualsInitializer import org.eclipse.cdt.core.dom.ast.IASTInitializer @@ -41,10 +48,7 @@ class InitializerHandler(lang: CXXLanguageFrontend) : override fun handleNode(node: IASTInitializer): Expression { return when (node) { is IASTEqualsInitializer -> handleEqualsInitializer(node) - // TODO: Initializer List is handled in ExpressionsHandler that actually handles - // InitializerClauses often used where - // one expects an expression. - is IASTInitializerList -> frontend.expressionHandler.handle(node) as Expression + is IASTInitializerList -> handleInitializerList(node) is CPPASTConstructorInitializer -> handleConstructorInitializer(node) else -> { return handleNotSupported(node, node.javaClass.name) @@ -54,6 +58,8 @@ class InitializerHandler(lang: CXXLanguageFrontend) : private fun handleConstructorInitializer(ctx: CPPASTConstructorInitializer): Expression { val constructExpression = newConstructExpression(ctx.rawSignature) + constructExpression.type = + (frontend.declaratorHandler.lastNode as? VariableDeclaration)?.type ?: unknownType() for ((i, argument) in ctx.arguments.withIndex()) { val arg = frontend.expressionHandler.handle(argument) @@ -66,6 +72,28 @@ class InitializerHandler(lang: CXXLanguageFrontend) : return constructExpression } + private fun handleInitializerList(ctx: IASTInitializerList): InitializerListExpression { + // Because an initializer list expression is used for many different things, it is important + // for us to know which kind of variable (or rather of which kind), we are initializing. + // This information can be found in the lastNode property of our declarator handler. + val targetType = + (frontend.declaratorHandler.lastNode as? ValueDeclaration)?.type ?: unknownType() + + val expression = newInitializerListExpression(targetType, ctx.rawSignature) + + for (clause in ctx.clauses) { + frontend.expressionHandler.handle(clause)?.let { + val edge = PropertyEdge(expression, it) + edge.addProperty(Properties.INDEX, expression.initializerEdges.size) + + expression.initializerEdges.add(edge) + expression.addPrevDFG(it) + } + } + + return expression + } + private fun handleEqualsInitializer(ctx: IASTEqualsInitializer): Expression { return frontend.expressionHandler.handle(ctx.initializerClause) ?: return newProblemExpression("could not parse initializer clause") diff --git a/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/PerformanceRegressionTest.kt b/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/PerformanceRegressionTest.kt index 03b473a6af..c8b4fcb8ae 100644 --- a/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/PerformanceRegressionTest.kt +++ b/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/PerformanceRegressionTest.kt @@ -73,7 +73,6 @@ class PerformanceRegressionTest { // enough for those special moments where for some reasons the GitHub runners // are slowing down (maybe because of some hidden quota). it.useParallelFrontends(false) - it.typeSystemActiveInFrontend(true) } assertNotNull(tu) } diff --git a/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/calls/FunctionPointerTest.kt b/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/calls/FunctionPointerTest.kt index 7ac666565a..a911cf74a5 100644 --- a/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/calls/FunctionPointerTest.kt +++ b/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/calls/FunctionPointerTest.kt @@ -102,7 +102,9 @@ internal class FunctionPointerTest : BaseTest() { "no_param", "no_param_uninitialized", "no_param_field", - "no_param_field_uninitialized" -> assertEquals(listOf(noParam), call.invokes) + "no_param_field_uninitialized" -> { + assertEquals(listOf(noParam), call.invokes) + } "single_param", "single_param_uninitialized", "single_param_field", diff --git a/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/templates/FunctionTemplateTest.kt b/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/templates/FunctionTemplateTest.kt index 102ae391e7..a9dfa42cd6 100644 --- a/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/templates/FunctionTemplateTest.kt +++ b/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/templates/FunctionTemplateTest.kt @@ -38,10 +38,7 @@ import de.fraunhofer.aisec.cpg.graph.statements.expressions.* import de.fraunhofer.aisec.cpg.graph.types.* import java.nio.file.Path import java.util.function.Predicate -import kotlin.test.Test -import kotlin.test.assertEquals -import kotlin.test.assertNotNull -import kotlin.test.assertTrue +import kotlin.test.* internal class FunctionTemplateTest : BaseTest() { private val topLevel = Path.of("src", "test", "resources", "templates", "functiontemplates") @@ -55,18 +52,17 @@ internal class FunctionTemplateTest : BaseTest() { topLevel, true ) - val variableDeclarations = result.variables - val x = findByUniqueName(variableDeclarations, "x") - assertEquals(UnknownType.getUnknownType(CPPLanguage()), x.type) - - val declaredReferenceExpressions = result.refs - val xDeclaredReferenceExpression = findByUniqueName(declaredReferenceExpressions, "x") - assertEquals(UnknownType.getUnknownType(CPPLanguage()), xDeclaredReferenceExpression.type) - - val binaryOperators = result.allChildren() - val dependentOperation = - findByUniquePredicate(binaryOperators) { b: BinaryOperator -> b.code == "val * N" } - assertEquals(UnknownType.getUnknownType(CPPLanguage()), dependentOperation.type) + val x = result.variables["x"] + assertNotNull(x) + assertIs(x.type) + + val xRef = result.refs["x"] + assertNotNull(xRef) + assertIs(xRef.type) + + val binOp = result.allChildren()[{ it.code == "val * N" }] + assertNotNull(binOp) + assertIs(binOp.type) } private fun testFunctionTemplateArguments( diff --git a/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/types/TypeTests.kt b/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/types/TypeTests.kt index 57efa21b86..12104eccd2 100644 --- a/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/types/TypeTests.kt +++ b/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/types/TypeTests.kt @@ -268,11 +268,11 @@ internal class TypeTests : BaseTest() { assertTrue(ptr.type is PointerType) assertEquals((ptr.type as PointerType).elementType, regularInt.type) - // Test type Propagation (auto) UnknownType + // Unresolved auto type propagation val unknown = findByUniqueName(variableDeclarations, "unknown") - assertEquals(UnknownType.getUnknownType(CPPLanguage()), unknown.type) + assertIs(unknown.type) - // Test type Propagation auto + // Resolved auto type propagation val propagated = findByUniqueName(variableDeclarations, "propagated") assertEquals(regularInt.type, propagated.type) } diff --git a/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CXXLanguageFrontendTest.kt b/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CXXLanguageFrontendTest.kt index d5fd09b57b..0a0434d109 100644 --- a/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CXXLanguageFrontendTest.kt +++ b/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CXXLanguageFrontendTest.kt @@ -54,7 +54,7 @@ internal class CXXLanguageFrontendTest : BaseTest() { @Test @Throws(Exception::class) fun testForEach() { - val file = File("src/test/resources/components/foreachstmt.cpp") + val file = File("src/test/resources/cxx/foreachstmt.cpp") val tu = analyzeAndGetFirstTU(listOf(file), file.parentFile.toPath(), true) val main = tu.getDeclarationsByName("main", FunctionDeclaration::class.java) assertFalse(main.isEmpty()) @@ -80,7 +80,7 @@ internal class CXXLanguageFrontendTest : BaseTest() { val i = stmt.singleDeclaration as VariableDeclaration assertNotNull(i) assertLocalName("i", i) - assertEquals(UnknownType.getUnknownType(CPPLanguage()), i.type) + assertIs(i.type) } @Test @@ -157,21 +157,17 @@ internal class CXXLanguageFrontendTest : BaseTest() { @Test @Throws(Exception::class) fun testCast() { - val file = File("src/test/resources/components/castexpr.cpp") + val file = File("src/test/resources/cxx/castexpr.cpp") val tu = analyzeAndGetFirstTU(listOf(file), file.parentFile.toPath(), true) with(tu) { - val main = tu.getDeclarationAs(0, FunctionDeclaration::class.java) - val e = - Objects.requireNonNull( - main!!.getBodyStatementAs(0, DeclarationStatement::class.java) - ) - ?.singleDeclaration as VariableDeclaration + val main = tu.functions["main"] + assertNotNull(main) + + val e = main.variables["e"] assertNotNull(e) assertEquals(objectType("ExtendedClass").pointer(), e.type) - val b = - Objects.requireNonNull(main.getBodyStatementAs(1, DeclarationStatement::class.java)) - ?.singleDeclaration as VariableDeclaration + val b = main.variables["b"] assertNotNull(b) assertEquals(objectType("BaseClass").pointer(), b.type) @@ -180,21 +176,19 @@ internal class CXXLanguageFrontendTest : BaseTest() { assertNotNull(cast) assertEquals(objectType("BaseClass").pointer(), cast.castType) - val staticCast = main.getBodyStatementAs(2, BinaryOperator::class.java) + val staticCast = main.getBodyStatementAs(2, AssignExpression::class.java) assertNotNull(staticCast) - cast = staticCast.rhs as CastExpression + cast = staticCast.rhs() assertNotNull(cast) assertLocalName("static_cast", cast) - val reinterpretCast = main.getBodyStatementAs(3, BinaryOperator::class.java) + val reinterpretCast = main.getBodyStatementAs(3, AssignExpression::class.java) assertNotNull(reinterpretCast) - cast = reinterpretCast.rhs as CastExpression + cast = reinterpretCast.rhs() assertNotNull(cast) assertLocalName("reinterpret_cast", cast) - val d = - Objects.requireNonNull(main.getBodyStatementAs(4, DeclarationStatement::class.java)) - ?.singleDeclaration as VariableDeclaration + val d = main.variables["d"] assertNotNull(d) cast = d.initializer as? CastExpression @@ -508,47 +502,45 @@ internal class CXXLanguageFrontendTest : BaseTest() { @Test @Throws(Exception::class) fun testAssignmentExpression() { - val file = File("src/test/resources/assignmentexpression.cpp") - val declaration = analyzeAndGetFirstTU(listOf(file), file.parentFile.toPath(), true) + val file = File("src/test/resources/cxx/assignmentexpression.cpp") + val tu = analyzeAndGetFirstTU(listOf(file), file.parentFile.toPath(), true) // just take a look at the second function - val functionDeclaration = declaration.getDeclarationAs(1, FunctionDeclaration::class.java) - val statements = functionDeclaration?.statements + val main = tu.functions["main"] + assertNotNull(main) + + val statements = main.statements assertNotNull(statements) - val declareA = statements[0] - val a = (declareA as DeclarationStatement).singleDeclaration + val a = main.variables["a"] val assignA = statements[1] - assertTrue(assignA is BinaryOperator) + assertTrue(assignA is AssignExpression) - var lhs = assignA.lhs - var rhs = assignA.rhs + var lhs = assignA.lhs() + var rhs = assignA.rhs() assertLocalName("a", lhs) assertEquals(2, (rhs as? Literal<*>)?.value) - assertRefersTo(assignA.lhs, a) - - val declareB = statements[2] - assertTrue(declareB is DeclarationStatement) + assertRefersTo(lhs, a) - val b = declareB.singleDeclaration + val b = main.variables["b"] // a = b val assignB = statements[3] - assertTrue(assignB is BinaryOperator) + assertTrue(assignB is AssignExpression) - lhs = assignB.lhs - rhs = assignB.rhs + lhs = assignB.lhs() + rhs = assignB.rhs() assertLocalName("a", lhs) assertTrue(rhs is DeclaredReferenceExpression) assertLocalName("b", rhs) assertRefersTo(rhs, b) val assignBWithFunction = statements[4] - assertTrue(assignBWithFunction is BinaryOperator) - assertLocalName("a", assignBWithFunction.lhs) - assertTrue(assignBWithFunction.rhs is CallExpression) + assertTrue(assignBWithFunction is AssignExpression) + assertLocalName("a", assignBWithFunction.lhs()) - val call = assignBWithFunction.rhs as CallExpression + val call = assignBWithFunction.rhs() + assertNotNull(call) assertLocalName("someFunction", call) assertRefersTo(call.arguments[0], b) } @@ -620,8 +612,10 @@ internal class CXXLanguageFrontendTest : BaseTest() { assertNotNull(statement) // b = *ptr; - val assign = statements[++line] as BinaryOperator - val dereference = assign.rhs as UnaryOperator + val assign = statements[++line] as AssignExpression + + val dereference = assign.rhs() + assertNotNull(dereference) input = dereference.input assertLocalName("ptr", input) assertEquals("*", dereference.operatorCode) @@ -631,51 +625,63 @@ internal class CXXLanguageFrontendTest : BaseTest() { @Test @Throws(Exception::class) fun testBinaryOperator() { - val file = File("src/test/resources/binaryoperator.cpp") + val file = File("src/test/resources/cxx/binaryoperator.cpp") val tu = analyzeAndGetFirstTU(listOf(file), file.parentFile.toPath(), true) - val statements = tu.getDeclarationAs(0, FunctionDeclaration::class.java)?.statements + + val main = tu.functions["main"] + assertNotNull(main) + + val statements = main.statements assertNotNull(statements) // first two statements are just declarations // a = b * 2 - var operator = statements[2] as? BinaryOperator - assertNotNull(operator) - assertLocalName("a", operator.lhs) - assertTrue(operator.rhs is BinaryOperator) + var assign = statements[2] as? AssignExpression + assertNotNull(assign) + + var ref = assign.lhs() + assertNotNull(ref) + assertLocalName("a", ref) - var rhs = operator.rhs as BinaryOperator - assertTrue(rhs.lhs is DeclaredReferenceExpression) - assertLocalName("b", rhs.lhs) - assertTrue(rhs.rhs is Literal<*>) - assertEquals(2, (rhs.rhs as Literal<*>).value) + var binOp = assign.rhs() + assertNotNull(binOp) + + assertTrue(binOp.lhs is DeclaredReferenceExpression) + assertLocalName("b", binOp.lhs) + assertTrue(binOp.rhs is Literal<*>) + assertEquals(2, (binOp.rhs as Literal<*>).value) // a = 1 * 1 - operator = statements[3] as? BinaryOperator - assertNotNull(operator) - assertLocalName("a", operator.lhs) - assertTrue(operator.rhs is BinaryOperator) + assign = statements[3] as? AssignExpression + assertNotNull(assign) + + ref = assign.lhs() + assertNotNull(ref) + assertLocalName("a", ref) - rhs = operator.rhs as BinaryOperator - assertTrue(rhs.lhs is Literal<*>) - assertEquals(1, (rhs.lhs as Literal<*>).value) - assertTrue(rhs.rhs is Literal<*>) - assertEquals(1, (rhs.rhs as Literal<*>).value) + binOp = assign.rhs() + assertNotNull(binOp) + + assertTrue(binOp.lhs is Literal<*>) + assertEquals(1, (binOp.lhs as Literal<*>).value) + assertTrue(binOp.rhs is Literal<*>) + assertEquals(1, (binOp.rhs as Literal<*>).value) // std::string* notMultiplication // this is not a multiplication, but a variable declaration with a pointer type, but - // syntactically no different than the previous ones + // syntactically no different from the previous ones val stmt = statements[4] as DeclarationStatement val decl = stmt.singleDeclaration as VariableDeclaration with(tu) { assertEquals(objectType("std::string").pointer(), decl.type) } assertLocalName("notMultiplication", decl) assertTrue(decl.initializer is BinaryOperator) - operator = decl.initializer as? BinaryOperator - assertNotNull(operator) - assertTrue(operator.lhs is Literal<*>) - assertEquals(0, (operator.lhs as Literal<*>).value) - assertTrue(operator.rhs is Literal<*>) - assertEquals(0, (operator.rhs as Literal<*>).value) + binOp = decl.initializer as? BinaryOperator + assertNotNull(binOp) + assertTrue(binOp.lhs is Literal<*>) + assertEquals(0, (binOp.lhs as Literal<*>).value) + assertTrue(binOp.rhs is Literal<*>) + assertEquals(0, (binOp.rhs as Literal<*>).value) } @Test @@ -1090,7 +1096,7 @@ internal class CXXLanguageFrontendTest : BaseTest() { @Test @Throws(Exception::class) fun testLocation() { - val file = File("src/test/resources/components/foreachstmt.cpp") + val file = File("src/test/resources/cxx/foreachstmt.cpp") val tu = analyzeAndGetFirstTU(listOf(file), file.parentFile.toPath(), true) val main = tu.functions["main"] assertNotNull(main) @@ -1140,7 +1146,7 @@ internal class CXXLanguageFrontendTest : BaseTest() { .sourceLocations(listOf(file)) .topLevel(file.parentFile) .defaultPasses() - .defaultLanguages() + .registerLanguage() .processAnnotations(true) .symbols( mapOf( @@ -1197,7 +1203,7 @@ internal class CXXLanguageFrontendTest : BaseTest() { .useUnityBuild(true) .loadIncludes(true) .defaultPasses() - .defaultLanguages() + .registerLanguage() ) assertEquals(1, declarations.size) // should contain 3 declarations (2 include and 1 function decl from the include) @@ -1325,16 +1331,16 @@ internal class CXXLanguageFrontendTest : BaseTest() { val file = File("src/test/resources/c/struct.c") val tu = analyzeAndGetFirstTU(listOf(file), file.parentFile.toPath(), true) - val main = tu.byNameOrNull("main") + val main = tu.functions["main"] assertNotNull(main) - val myStruct = tu.byNameOrNull("MyStruct") + val myStruct = tu.records["MyStruct"] assertNotNull(myStruct) - val field = myStruct.byNameOrNull("field") + val field = myStruct.fields["field"] assertNotNull(field) - val s = main.bodyOrNull()?.singleDeclaration as? VariableDeclaration + val s = main.variables["s"] assertNotNull(s) assertEquals(myStruct, (s.type as? ObjectType)?.recordDeclaration) @@ -1574,4 +1580,16 @@ internal class CXXLanguageFrontendTest : BaseTest() { assertLocalName("int", tu.functions["main"]?.returnTypes?.firstOrNull()) } + + @Test + @Throws(Exception::class) + fun testFancyTypes() { + val file = File("src/test/resources/cxx/fancy_types.cpp") + val tu = analyzeAndGetFirstTU(listOf(file), file.parentFile.toPath(), true) + assertNotNull(tu) + + val ptr = tu.variables["ptr"] + assertNotNull(ptr) + assertLocalName("decltype(nullptr)", ptr.type) + } } diff --git a/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/LambdaTest.kt b/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/LambdaTest.kt index eb970ae368..f223455397 100644 --- a/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/LambdaTest.kt +++ b/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/LambdaTest.kt @@ -29,6 +29,7 @@ import de.fraunhofer.aisec.cpg.TranslationConfiguration import de.fraunhofer.aisec.cpg.TranslationManager import de.fraunhofer.aisec.cpg.graph.* import de.fraunhofer.aisec.cpg.graph.statements.expressions.* +import de.fraunhofer.aisec.cpg.graph.types.FunctionPointerType import java.io.File import kotlin.test.* @@ -40,7 +41,7 @@ class CPPLambdaTest { TranslationConfiguration.builder() .sourceLocations(File("src/test/resources/cxx/lambdas.cpp")) .defaultPasses() - .defaultLanguages() + .registerLanguage() .build() val analyzer = TranslationManager.builder().config(config).build() val result = analyzer.analyze().get() @@ -51,10 +52,12 @@ class CPPLambdaTest { val lambdaVar = function.variables["this_is_a_lambda"] assertNotNull(lambdaVar) + assertIs(lambdaVar.type) + val lambda = lambdaVar.initializer as? LambdaExpression assertNotNull(lambda) - assertTrue(lambda in lambdaVar.prevEOG) + val printFunctionCall = function.calls["print_function"] assertNotNull(printFunctionCall) assertTrue(printFunctionCall in lambda.prevEOG) @@ -70,7 +73,7 @@ class CPPLambdaTest { TranslationConfiguration.builder() .sourceLocations(File("src/test/resources/cxx/lambdas.cpp")) .defaultPasses() - .defaultLanguages() + .registerLanguage() .build() val analyzer = TranslationManager.builder().config(config).build() val result = analyzer.analyze().get() @@ -104,7 +107,7 @@ class CPPLambdaTest { TranslationConfiguration.builder() .sourceLocations(File("src/test/resources/cxx/lambdas.cpp")) .defaultPasses() - .defaultLanguages() + .registerLanguage() .build() val analyzer = TranslationManager.builder().config(config).build() val result = analyzer.analyze().get() @@ -131,7 +134,7 @@ class CPPLambdaTest { TranslationConfiguration.builder() .sourceLocations(File("src/test/resources/cxx/lambdas.cpp")) .defaultPasses() - .defaultLanguages() + .registerLanguage() .build() val analyzer = TranslationManager.builder().config(config).build() val result = analyzer.analyze().get() @@ -167,7 +170,7 @@ class CPPLambdaTest { TranslationConfiguration.builder() .sourceLocations(File("src/test/resources/cxx/lambdas.cpp")) .defaultPasses() - .defaultLanguages() + .registerLanguage() .build() val analyzer = TranslationManager.builder().config(config).build() val result = analyzer.analyze().get() @@ -204,7 +207,7 @@ class CPPLambdaTest { TranslationConfiguration.builder() .sourceLocations(File("src/test/resources/cxx/lambdas.cpp")) .defaultPasses() - .defaultLanguages() + .registerLanguage() .build() val analyzer = TranslationManager.builder().config(config).build() val result = analyzer.analyze().get() @@ -240,7 +243,7 @@ class CPPLambdaTest { TranslationConfiguration.builder() .sourceLocations(File("src/test/resources/cxx/lambdas.cpp")) .defaultPasses() - .defaultLanguages() + .registerLanguage() .build() val analyzer = TranslationManager.builder().config(config).build() val result = analyzer.analyze().get() @@ -274,7 +277,7 @@ class CPPLambdaTest { TranslationConfiguration.builder() .sourceLocations(File("src/test/resources/cxx/lambdas.cpp")) .defaultPasses() - .defaultLanguages() + .registerLanguage() .build() val analyzer = TranslationManager.builder().config(config).build() val result = analyzer.analyze().get() @@ -310,7 +313,7 @@ class CPPLambdaTest { TranslationConfiguration.builder() .sourceLocations(File("src/test/resources/cxx/lambdas.cpp")) .defaultPasses() - .defaultLanguages() + .registerLanguage() .build() val analyzer = TranslationManager.builder().config(config).build() val result = analyzer.analyze().get() @@ -337,7 +340,7 @@ class CPPLambdaTest { TranslationConfiguration.builder() .sourceLocations(File("src/test/resources/cxx/lambdas.cpp")) .defaultPasses() - .defaultLanguages() + .registerLanguage() .build() val analyzer = TranslationManager.builder().config(config).build() val result = analyzer.analyze().get() @@ -358,7 +361,7 @@ class CPPLambdaTest { TranslationConfiguration.builder() .sourceLocations(File("src/test/resources/cxx/lambdas.cpp")) .defaultPasses() - .defaultLanguages() + .registerLanguage() .build() val analyzer = TranslationManager.builder().config(config).build() val result = analyzer.analyze().get() diff --git a/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/helpers/BenchmarkCXXTest.kt b/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/helpers/BenchmarkCXXTest.kt index d0e5b5cbe4..1d283bf9b5 100644 --- a/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/helpers/BenchmarkCXXTest.kt +++ b/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/helpers/BenchmarkCXXTest.kt @@ -37,7 +37,7 @@ class BenchmarkCXXTest { @Test fun testGetBenchmarkResult() { - val file = File("src/test/resources/components/foreachstmt.cpp") + val file = File("src/test/resources/cxx/foreachstmt.cpp") val tr = TestUtils.analyze(listOf(file), file.parentFile.toPath(), true) assertNotNull(tr) @@ -58,7 +58,7 @@ class BenchmarkCXXTest { @Test fun testPrintBenchmark() { - val file = File("src/test/resources/components/foreachstmt.cpp") + val file = File("src/test/resources/cxx/foreachstmt.cpp") val tr = TestUtils.analyze(listOf(file), file.parentFile.toPath(), true) assertNotNull(tr) diff --git a/cpg-language-cxx/src/test/resources/assignmentexpression.cpp b/cpg-language-cxx/src/test/resources/cxx/assignmentexpression.cpp similarity index 100% rename from cpg-language-cxx/src/test/resources/assignmentexpression.cpp rename to cpg-language-cxx/src/test/resources/cxx/assignmentexpression.cpp diff --git a/cpg-language-cxx/src/test/resources/binaryoperator.cpp b/cpg-language-cxx/src/test/resources/cxx/binaryoperator.cpp similarity index 100% rename from cpg-language-cxx/src/test/resources/binaryoperator.cpp rename to cpg-language-cxx/src/test/resources/cxx/binaryoperator.cpp diff --git a/cpg-language-cxx/src/test/resources/components/castexpr.cpp b/cpg-language-cxx/src/test/resources/cxx/castexpr.cpp similarity index 100% rename from cpg-language-cxx/src/test/resources/components/castexpr.cpp rename to cpg-language-cxx/src/test/resources/cxx/castexpr.cpp diff --git a/cpg-language-cxx/src/test/resources/cxx/fancy_types.cpp b/cpg-language-cxx/src/test/resources/cxx/fancy_types.cpp new file mode 100644 index 0000000000..b9851fe027 --- /dev/null +++ b/cpg-language-cxx/src/test/resources/cxx/fancy_types.cpp @@ -0,0 +1,3 @@ +typedef __decltype(nullptr) nullptr_t; + +nullptr_t ptr; diff --git a/cpg-language-cxx/src/test/resources/components/foreachstmt.cpp b/cpg-language-cxx/src/test/resources/cxx/foreachstmt.cpp similarity index 100% rename from cpg-language-cxx/src/test/resources/components/foreachstmt.cpp rename to cpg-language-cxx/src/test/resources/cxx/foreachstmt.cpp diff --git a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/ExpressionHandler.kt b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/ExpressionHandler.kt index 996360809d..6d268e6b2e 100644 --- a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/ExpressionHandler.kt +++ b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/ExpressionHandler.kt @@ -238,7 +238,7 @@ class ExpressionHandler(frontend: GoLanguageFrontend) : // Pass the remaining arguments for (expr in args.subList(1.coerceAtMost(args.size - 1), args.size - 1)) { - handle(expr)?.let { construct += it } + handle(expr).let { construct += it } } construct @@ -347,7 +347,7 @@ class ExpressionHandler(frontend: GoLanguageFrontend) : val construct = newConstructExpression(type.name, rawNode = compositeLit) construct.type = type - val list = newInitializerListExpression(rawNode = compositeLit) + val list = newInitializerListExpression(type, rawNode = compositeLit) construct += list // Normally, the construct expression would not have DFG edge, but in this case we are diff --git a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoHandler.kt b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoHandler.kt index cfac505f13..dc21160c7a 100644 --- a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoHandler.kt +++ b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoHandler.kt @@ -53,6 +53,8 @@ abstract class GoHandler, oldType: Type) { + (forEach.iterable as HasType).registerTypeObserver( + object : HasType.TypeObserver { + override fun typeChanged(newType: Type, src: HasType) { if (src.type is UnknownType) { return } @@ -154,7 +155,7 @@ class GoExtraPass(ctx: TranslationContext) : ComponentPass(ctx), ScopeProvider { } } - override fun possibleSubTypesChanged(src: HasType, root: MutableList) { + override fun assignedTypeChanged(assignedTypes: Set, src: HasType) { // Nothing to do } } @@ -178,7 +179,7 @@ class GoExtraPass(ctx: TranslationContext) : ComponentPass(ctx), ScopeProvider { val ref = scopeManager.resolveReference(expr) if (ref == null) { // We need to implicitly declare it, if its not declared before. - val decl = newVariableDeclaration(expr.name, expr.type) + val decl = newVariableDeclaration(expr.name, expr.autoType()) decl.location = expr.location decl.isImplicit = true decl.initializer = assign.findValue(expr) diff --git a/cpg-language-go/src/test/resources/golang/literal.go b/cpg-language-go/src/test/resources/golang/literal.go index 0450e78c4b..4b7207b8c0 100644 --- a/cpg-language-go/src/test/resources/golang/literal.go +++ b/cpg-language-go/src/test/resources/golang/literal.go @@ -4,8 +4,9 @@ const a = 1 const s = "test" const f = 1.0 const f32 float32 = 1.00 + var n *int = nil var fn = func(_ int) int { - return 1 -} \ No newline at end of file + return 1 +} diff --git a/cpg-language-java/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/java/ExpressionHandler.kt b/cpg-language-java/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/java/ExpressionHandler.kt index 2b0978b562..5950b651d6 100644 --- a/cpg-language-java/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/java/ExpressionHandler.kt +++ b/cpg-language-java/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/java/ExpressionHandler.kt @@ -28,6 +28,7 @@ package de.fraunhofer.aisec.cpg.frontends.java import com.github.javaparser.Range import com.github.javaparser.TokenRange import com.github.javaparser.ast.Node +import com.github.javaparser.ast.body.VariableDeclarator import com.github.javaparser.ast.expr.* import com.github.javaparser.ast.expr.Expression import com.github.javaparser.resolution.UnsolvedSymbolException @@ -42,6 +43,7 @@ import de.fraunhofer.aisec.cpg.graph.statements.expressions.* import de.fraunhofer.aisec.cpg.graph.types.* import java.util.function.Supplier import kotlin.collections.set +import kotlin.jvm.optionals.getOrNull import org.slf4j.LoggerFactory class ExpressionHandler(lang: JavaLanguageFrontend) : @@ -95,7 +97,7 @@ class ExpressionHandler(lang: JavaLanguageFrontend) : castExpression.type = frontend.typeOf(castExpr.type.resolve().asPrimitive()) } else { // Get Runtime type from cast expression for complex types; - castExpression.expression.registerTypeListener(castExpression) + // castExpression.expression.registerTypeListener(castExpression) } return castExpression } @@ -130,9 +132,18 @@ class ExpressionHandler(lang: JavaLanguageFrontend) : private fun handleArrayInitializerExpr(expr: Expression): Statement { val arrayInitializerExpr = expr as ArrayInitializerExpr + + // We need to go back to the parent to get the array type + val arrayType = + when (val parent = expr.parentNode.getOrNull()) { + is ArrayCreationExpr -> frontend.typeOf(parent.elementType).array() + is VariableDeclarator -> frontend.typeOf(parent.type) + else -> unknownType() + } + // ArrayInitializerExpressions are converted into InitializerListExpressions to reduce the // syntactic distance a CPP and JAVA CPG - val initList = this.newInitializerListExpression(expr.toString()) + val initList = this.newInitializerListExpression(arrayType, expr.toString()) val initializers = arrayInitializerExpr.values .map { handle(it) } @@ -196,7 +207,7 @@ class ExpressionHandler(lang: JavaLanguageFrontend) : return this.newConditionalExpression(condition, thenExpr, elseExpr, superType) } - private fun handleAssignmentExpression(expr: Expression): BinaryOperator { + private fun handleAssignmentExpression(expr: Expression): AssignExpression { val assignExpr = expr.asAssignExpr() // first, handle the target. this is the first argument of the operator call @@ -210,11 +221,15 @@ class ExpressionHandler(lang: JavaLanguageFrontend) : handle(assignExpr.value) as? de.fraunhofer.aisec.cpg.graph.statements.expressions.Expression ?: newProblemExpression("could not parse lhs") - val binaryOperator = - this.newBinaryOperator(assignExpr.operator.asString(), assignExpr.toString()) - binaryOperator.lhs = lhs - binaryOperator.rhs = rhs - return binaryOperator + val assign = + this.newAssignExpression( + assignExpr.operator.asString(), + listOf(lhs), + listOf(rhs), + rawNode = assignExpr + ) + + return assign } // Not sure how to handle this exactly yet diff --git a/cpg-language-java/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/java/StatementHandler.kt b/cpg-language-java/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/java/StatementHandler.kt index adffc066ac..4dd85bb37e 100644 --- a/cpg-language-java/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/java/StatementHandler.kt +++ b/cpg-language-java/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/java/StatementHandler.kt @@ -569,7 +569,7 @@ class StatementHandler(lang: JavaLanguageFrontend?) : ): de.fraunhofer.aisec.cpg.graph.statements.CatchClause { val cClause = this.newCatchClause(catchCls.toString()) frontend.scopeManager.enterScope(cClause) - val possibleTypes: MutableList = ArrayList() + val possibleTypes = mutableSetOf() val concreteType: Type if (catchCls.parameter.type is UnionType) { for (t in (catchCls.parameter.type as UnionType).elements) { @@ -590,7 +590,7 @@ class StatementHandler(lang: JavaLanguageFrontend?) : catchCls.parameter.toString(), false ) - parameter.possibleSubTypes = possibleTypes + parameter.addAssignedTypes(possibleTypes) val body = handleBlockStatement(catchCls.body) cClause.body = body cClause.parameter = parameter diff --git a/cpg-language-java/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/JavaCallResolverHelper.kt b/cpg-language-java/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/JavaCallResolverHelper.kt index d2940fa7dd..d1ddc7973a 100644 --- a/cpg-language-java/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/JavaCallResolverHelper.kt +++ b/cpg-language-java/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/JavaCallResolverHelper.kt @@ -34,6 +34,7 @@ import de.fraunhofer.aisec.cpg.graph.objectType import de.fraunhofer.aisec.cpg.graph.statements.expressions.DeclaredReferenceExpression import de.fraunhofer.aisec.cpg.graph.statements.expressions.MemberCallExpression import de.fraunhofer.aisec.cpg.graph.statements.expressions.MemberExpression +import de.fraunhofer.aisec.cpg.graph.types.HasType import de.fraunhofer.aisec.cpg.helpers.Util import de.fraunhofer.aisec.cpg.passes.CallResolver.Companion.LOGGER @@ -91,12 +92,17 @@ class JavaCallResolverHelper { if (target != null) { val superType = target.toType() // Explicitly set the type of the call's base to the super type, basically "casting" - // the - // "this" object to the super class + // the "this" object to the super class callee.base.type = superType - // And set the possible subtypes, to ensure, that really only our super type is in - // there - callee.base.updatePossibleSubtypes(listOf(superType)) + + val refersTo = (callee.base as? DeclaredReferenceExpression)?.refersTo + if (refersTo is HasType) { + refersTo.type = superType + refersTo.assignedTypes = mutableSetOf(superType) + } + + // Make sure that really only our super class is in the list of assigned types + callee.base.assignedTypes = mutableSetOf(superType) return true } diff --git a/cpg-language-java/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/java/JavaLanguageFrontendTest.kt b/cpg-language-java/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/java/JavaLanguageFrontendTest.kt index f538719246..ccc0f737ad 100644 --- a/cpg-language-java/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/java/JavaLanguageFrontendTest.kt +++ b/cpg-language-java/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/java/JavaLanguageFrontendTest.kt @@ -627,10 +627,10 @@ internal class JavaLanguageFrontendTest : BaseTest() { assertNotNull(record) val constructor = record.constructors[0] - val op = constructor.getBodyStatementAs(0, BinaryOperator::class.java) + val op = constructor.getBodyStatementAs(0, AssignExpression::class.java) assertNotNull(op) - val lhs = op.lhs as? MemberExpression + val lhs = op.lhs() val receiver = (lhs?.base as? DeclaredReferenceExpression)?.refersTo as? VariableDeclaration? assertNotNull(receiver) @@ -768,10 +768,10 @@ internal class JavaLanguageFrontendTest : BaseTest() { val doSomething = evenMoreInnerClass.methods["doSomething"] assertNotNull(doSomething) - val binOp = doSomething.bodyOrNull() - assertNotNull(binOp) + val assign = doSomething.bodyOrNull() + assertNotNull(assign) - val ref = ((binOp.rhs as? MemberExpression)?.base as DeclaredReferenceExpression).refersTo + val ref = ((assign.rhs())?.base as DeclaredReferenceExpression).refersTo assertNotNull(ref) assertSame(ref, thisOuterClass) } diff --git a/cpg-language-java/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/types/TypeTests.kt b/cpg-language-java/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/types/TypeTests.kt index 5302853f54..6d89f197a0 100644 --- a/cpg-language-java/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/types/TypeTests.kt +++ b/cpg-language-java/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/types/TypeTests.kt @@ -57,14 +57,14 @@ internal class TypeTests : BaseTest() { // Type of field t val fieldDeclarations = result.fields val fieldDeclarationT = findByUniqueName(fieldDeclarations, "t") - assertTrue(fieldDeclarationT.possibleSubTypes.contains(typeT)) + assertTrue(fieldDeclarationT.assignedTypes.contains(typeT)) // Parameter of set Method val methodDeclarations = result.methods val methodDeclarationSet = findByUniqueName(methodDeclarations, "set") val t = methodDeclarationSet.parameters[0] assertEquals(typeT, t.type) - assertTrue(t.possibleSubTypes.contains(typeT)) + assertTrue(t.assignedTypes.contains(typeT)) // Return Type of get Method val methodDeclarationGet = findByUniqueName(methodDeclarations, "get") diff --git a/cpg-language-llvm/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/llvm/ExpressionHandler.kt b/cpg-language-llvm/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/llvm/ExpressionHandler.kt index 72be04e5b3..f38f01ff39 100644 --- a/cpg-language-llvm/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/llvm/ExpressionHandler.kt +++ b/cpg-language-llvm/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/llvm/ExpressionHandler.kt @@ -312,8 +312,9 @@ class ExpressionHandler(lang: LLVMIRLanguageFrontend) : return newLiteral(string, frontend.typeOf(valueRef), frontend.codeOf(valueRef)) } - val list = newInitializerListExpression(frontend.codeOf(valueRef)) val arrayType = LLVMTypeOf(valueRef) + val list = + newInitializerListExpression(frontend.typeOf(valueRef), frontend.codeOf(valueRef)) val length = if (LLVMIsAConstantDataArray(valueRef) != null) { LLVMGetArrayLength(arrayType) diff --git a/cpg-language-llvm/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/llvm/StatementHandler.kt b/cpg-language-llvm/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/llvm/StatementHandler.kt index bb741ce560..93f6c86170 100644 --- a/cpg-language-llvm/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/llvm/StatementHandler.kt +++ b/cpg-language-llvm/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/llvm/StatementHandler.kt @@ -1291,7 +1291,7 @@ class StatementHandler(lang: LLVMIRLanguageFrontend) : private fun handleShufflevector(instr: LLVMValueRef): Statement { val instrStr = frontend.codeOf(instr) - val list = newInitializerListExpression(instrStr) + val list = newInitializerListExpression(frontend.typeOf(instr), instrStr) val elementType = frontend.typeOf(instr).dereference() val initializers = mutableListOf() @@ -1448,10 +1448,6 @@ class StatementHandler(lang: LLVMIRLanguageFrontend) : code ) (assignment.lhs.first() as DeclaredReferenceExpression).type = type - (assignment.lhs.first() as DeclaredReferenceExpression).unregisterTypeListener( - assignment - ) - assignment.unregisterTypeListener(assignment.lhs.first() as DeclaredReferenceExpression) (assignment.lhs.first() as DeclaredReferenceExpression).refersTo = declaration flatAST.add(assignment) diff --git a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonLanguage.kt b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonLanguage.kt index 15a48176d8..eb96e1abc0 100644 --- a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonLanguage.kt +++ b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonLanguage.kt @@ -81,20 +81,17 @@ class PythonLanguage : Language(), HasShortCircuitOperat val unknownType = UnknownType.getUnknownType(this) if ( operation.operatorCode == "/" && - operation.lhs.propagationType is NumericType && - operation.rhs.propagationType is NumericType + operation.lhs.type is NumericType && + operation.rhs.type is NumericType ) { // In Python, the / operation automatically casts the result to a float return getSimpleTypeOf("float") ?: unknownType } else if ( operation.operatorCode == "//" && - operation.lhs.propagationType is NumericType && - operation.rhs.propagationType is NumericType + operation.lhs.type is NumericType && + operation.rhs.type is NumericType ) { - return if ( - operation.lhs.propagationType is IntegerType && - operation.rhs.propagationType is IntegerType - ) { + return if (operation.lhs.type is IntegerType && operation.rhs.type is IntegerType) { // In Python, the // operation keeps the type as an int if both inputs are integers // or casts it to a float otherwise. getSimpleTypeOf("int") ?: unknownType diff --git a/cpg-language-python/src/main/python/CPGPython/_expressions.py b/cpg-language-python/src/main/python/CPGPython/_expressions.py index 366d917141..b7ee1506a8 100644 --- a/cpg-language-python/src/main/python/CPGPython/_expressions.py +++ b/cpg-language-python/src/main/python/CPGPython/_expressions.py @@ -107,7 +107,9 @@ def handle_expression_impl(self, expr): return r elif isinstance(expr, ast.Dict): ile = ExpressionBuilderKt.newInitializerListExpression( - self.frontend, self.get_src_code(expr)) + self.frontend, + UnknownType.getUnknownType(self.frontend.getLanguage()), + self.get_src_code(expr)) lst = [] @@ -357,7 +359,9 @@ def handle_expression_impl(self, expr): return r elif isinstance(expr, ast.List): ile = ExpressionBuilderKt.newInitializerListExpression( - self.frontend, self.get_src_code(expr)) + self.frontend, + UnknownType.getUnknownType(self.frontend.getLanguage()), + self.get_src_code(expr)) lst = [] @@ -370,7 +374,9 @@ def handle_expression_impl(self, expr): return ile elif isinstance(expr, ast.Tuple): ile = ExpressionBuilderKt.newInitializerListExpression( - self.frontend, self.get_src_code(expr)) + self.frontend, + UnknownType.getUnknownType(self.frontend.getLanguage()), + self.get_src_code(expr)) lst = [] diff --git a/cpg-language-python/src/main/python/CPGPython/_statements.py b/cpg-language-python/src/main/python/CPGPython/_statements.py index 5b27a9f519..40f7b98a48 100644 --- a/cpg-language-python/src/main/python/CPGPython/_statements.py +++ b/cpg-language-python/src/main/python/CPGPython/_statements.py @@ -31,6 +31,7 @@ from de.fraunhofer.aisec.cpg.graph import ExpressionBuilderKt from de.fraunhofer.aisec.cpg.graph.statements import CompoundStatement from de.fraunhofer.aisec.cpg.graph.types import UnknownType +from java.util import ArrayList import ast @@ -510,16 +511,20 @@ def handle_assign_impl(self, stmt): target = self.handle_expression(stmt.target) op = self.handle_operator_code(stmt.op) value = self.handle_expression(stmt.value) - r = ExpressionBuilderKt.newBinaryOperator(self.frontend, - op, self.get_src_code(stmt)) - r.setLhs(target) - r.setRhs(value) + lhs = ArrayList() + lhs.add(target) + rhs = ArrayList() + rhs.add(value) + r = ExpressionBuilderKt.newAssignExpression(self.frontend, + op, lhs, rhs, + self.get_src_code(stmt)) return r if isinstance(stmt, ast.Assign) and len(stmt.targets) != 1: self.log_with_loc(NOT_IMPLEMENTED_MSG, loglevel="ERROR") - r = ExpressionBuilderKt.newBinaryOperator(self.frontend, - "=", self.get_src_code(stmt) - ) + r = ExpressionBuilderKt.newAssignExpression(self.frontend, + "=", ArrayList(), + ArrayList(), + self.get_src_code(stmt)) return r if isinstance(stmt, ast.Assign): target = stmt.targets[0] @@ -539,9 +544,9 @@ def handle_assign_impl(self, stmt): "Expected a DeclaredReferenceExpression or MemberExpression " "but got \"%s\". Skipping." % lhs.java_name, loglevel="ERROR") - r = ExpressionBuilderKt.newBinaryOperator(self.frontend, - "=", - self.get_src_code(stmt)) + r = ExpressionBuilderKt.newArrayList(self.frontend, + "=", ArrayList(), ArrayList(), + self.get_src_code(stmt)) return r resolved_lhs = self.scopemanager.resolveReference(lhs) @@ -549,12 +554,16 @@ def handle_assign_impl(self, stmt): in_function = self.scopemanager.isInFunction() if resolved_lhs is not None: - # found var => BinaryOperator "=" - binop = ExpressionBuilderKt.newBinaryOperator( - self.frontend, "=", self.get_src_code(stmt)) - binop.setLhs(lhs) + lhsList = ArrayList() + lhsList.add(lhs) + rhsList = ArrayList() if rhs is not None: - binop.setRhs(rhs) + rhsList.add(rhs) + + # found var => BinaryOperator "=" + binop = ExpressionBuilderKt.newAssignExpression( + self.frontend, "=", lhsList, rhsList, self.get_src_code(stmt)) + return binop else: if in_record and not in_function: @@ -601,13 +610,13 @@ def bar(self): if rhs is not None: v = DeclarationBuilderKt.newVariableDeclaration( self.frontend, lhs.getName(), - rhs.getType(), + TypeBuilderKt.autoType(self.frontend), self.get_src_code(stmt), False) else: v = DeclarationBuilderKt.newVariableDeclaration( self.frontend, lhs.getName(), - TypeBuilderKt.unknownType(self.frontend), + TypeBuilderKt.autoType(self.frontend), self.get_src_code(stmt), False) if rhs is not None: @@ -661,13 +670,13 @@ def bar(self): if rhs is not None: v = DeclarationBuilderKt.newVariableDeclaration( self.frontend, lhs.getName(), - rhs.getType(), + TypeBuilderKt.autoType(self.frontend), self.get_src_code(stmt), False) else: v = DeclarationBuilderKt.newVariableDeclaration( self.frontend, lhs.getName(), - TypeBuilderKt.unknownType(self.frontend), + TypeBuilderKt.autoType(self.frontend), self.get_src_code(stmt), False) if rhs is not None: diff --git a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt index 39fd3815a6..ec275da66d 100644 --- a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt +++ b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonFrontendTest.kt @@ -593,11 +593,11 @@ class PythonFrontendTest : BaseTest() { val methBody = meth.body as? CompoundStatement assertNotNull(methBody) - val assign = methBody.statements[0] as? BinaryOperator + val assign = methBody.statements[0] as? AssignExpression assertNotNull(assign) - val assignLhs = assign.lhs as? MemberExpression - val assignRhs = assign.rhs as? BinaryOperator + val assignLhs = assign.lhs() + val assignRhs = assign.rhs() assertEquals("=", assign.operatorCode) assertNotNull(assignLhs) assertNotNull(assignRhs) @@ -657,15 +657,15 @@ class PythonFrontendTest : BaseTest() { assertNotNull(classFieldWithInit) // classFieldNoInitializer = classFieldWithInit - val assignClsFieldOutsideFunc = clsFoo.statements[2] as? BinaryOperator + val assignClsFieldOutsideFunc = clsFoo.statements[2] as? AssignExpression assertNotNull(assignClsFieldOutsideFunc) assertEquals( classFieldNoInitializer, - (assignClsFieldOutsideFunc.lhs as? DeclaredReferenceExpression)?.refersTo + (assignClsFieldOutsideFunc.lhs())?.refersTo ) assertEquals( classFieldWithInit, - (assignClsFieldOutsideFunc.rhs as? DeclaredReferenceExpression)?.refersTo + (assignClsFieldOutsideFunc.rhs())?.refersTo ) assertEquals("=", assignClsFieldOutsideFunc.operatorCode) @@ -680,41 +680,41 @@ class PythonFrontendTest : BaseTest() { assertNotNull(decl0.initializer) // self.classFieldNoInitializer = 789 - val barStmt1 = barBody.statements[1] as? BinaryOperator + val barStmt1 = barBody.statements[1] as? AssignExpression assertNotNull(barStmt1) - assertEquals(classFieldNoInitializer, (barStmt1.lhs as? MemberExpression)?.refersTo) + assertEquals(classFieldNoInitializer, (barStmt1.lhs())?.refersTo) // self.classFieldWithInit = 12 - val barStmt2 = barBody.statements[2] as? BinaryOperator + val barStmt2 = barBody.statements[2] as? AssignExpression assertNotNull(barStmt2) - assertEquals(classFieldWithInit, (barStmt2.lhs as? MemberExpression)?.refersTo) + assertEquals(classFieldWithInit, (barStmt2.lhs())?.refersTo) // classFieldNoInitializer = "shadowed" - val barStmt3 = barBody.statements[3] as? BinaryOperator + val barStmt3 = barBody.statements[3] as? AssignExpression assertNotNull(barStmt3) assertEquals("=", barStmt3.operatorCode) assertEquals( classFieldNoInitializer, - (barStmt3.lhs as? DeclaredReferenceExpression)?.refersTo + (barStmt3.lhs())?.refersTo ) - assertEquals("shadowed", (barStmt3.rhs as? Literal<*>)?.value) + assertEquals("shadowed", (barStmt3.rhs>())?.value) // classFieldWithInit = "shadowed" - val barStmt4 = barBody.statements[4] as? BinaryOperator + val barStmt4 = barBody.statements[4] as? AssignExpression assertNotNull(barStmt4) assertEquals("=", barStmt4.operatorCode) - assertEquals(classFieldWithInit, (barStmt4.lhs as? DeclaredReferenceExpression)?.refersTo) - assertEquals("shadowed", (barStmt4.rhs as? Literal<*>)?.value) + assertEquals(classFieldWithInit, (barStmt4.lhs())?.refersTo) + assertEquals("shadowed", (barStmt4.rhs>())?.value) // classFieldDeclaredInFunction = "shadowed" - val barStmt5 = barBody.statements[5] as? BinaryOperator + val barStmt5 = barBody.statements[5] as? AssignExpression assertNotNull(barStmt5) assertEquals("=", barStmt5.operatorCode) assertEquals( classFieldDeclaredInFunction, - (barStmt5.lhs as? DeclaredReferenceExpression)?.refersTo + (barStmt5.lhs())?.refersTo ) - assertEquals("shadowed", (barStmt5.rhs as? Literal<*>)?.value) + assertEquals("shadowed", (barStmt5.rhs>())?.value) /* TODO: foo = Foo() @@ -943,10 +943,10 @@ class PythonFrontendTest : BaseTest() { assertLocalName("z", elseStmt1) // phr = {**z, **content} - val elseStmt2 = ifElse.statements[1] as? BinaryOperator + val elseStmt2 = ifElse.statements(1) assertNotNull(elseStmt2) assertEquals("=", elseStmt2.operatorCode) - val elseStmt2Rhs = elseStmt2.rhs as? InitializerListExpression + val elseStmt2Rhs = elseStmt2.rhs() assertNotNull(elseStmt2Rhs) } diff --git a/cpg-language-typescript/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/typescript/ExpressionHandler.kt b/cpg-language-typescript/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/typescript/ExpressionHandler.kt index 9069f4910a..9b50a431f7 100644 --- a/cpg-language-typescript/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/typescript/ExpressionHandler.kt +++ b/cpg-language-typescript/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/typescript/ExpressionHandler.kt @@ -143,7 +143,7 @@ class ExpressionHandler(lang: TypeScriptLanguageFrontend) : } private fun handleObjectLiteralExpression(node: TypeScriptNode): InitializerListExpression { - val ile = newInitializerListExpression(this.frontend.codeOf(node)) + val ile = newInitializerListExpression(unknownType(), this.frontend.codeOf(node)) ile.initializers = node.children?.mapNotNull { this.handle(it) } ?: emptyList() diff --git a/cpg-language-typescript/src/main/nodejs/package-lock.json b/cpg-language-typescript/src/main/nodejs/package-lock.json index 1fe34fba97..06c54debc4 100644 --- a/cpg-language-typescript/src/main/nodejs/package-lock.json +++ b/cpg-language-typescript/src/main/nodejs/package-lock.json @@ -13,7 +13,7 @@ "@rollup/plugin-commonjs": "^25.0.3", "@rollup/plugin-node-resolve": "^15.1.0", "@rollup/plugin-typescript": "^11.1.2", - "rollup": "^3.27.0", + "rollup": "^3.26.3", "tslib": "^2.6.0" } }, diff --git a/cpg-neo4j/src/test/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/ApplicationTest.kt b/cpg-neo4j/src/test/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/ApplicationTest.kt index 33f38fe2cb..b7e26cecee 100644 --- a/cpg-neo4j/src/test/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/ApplicationTest.kt +++ b/cpg-neo4j/src/test/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/ApplicationTest.kt @@ -25,9 +25,11 @@ */ package de.fraunhofer.aisec.cpg_vis_neo4j -import de.fraunhofer.aisec.cpg.TranslationManager +import de.fraunhofer.aisec.cpg.* +import de.fraunhofer.aisec.cpg.graph.builder.* import de.fraunhofer.aisec.cpg.graph.declarations.FunctionDeclaration import de.fraunhofer.aisec.cpg.graph.functions +import de.fraunhofer.aisec.cpg.graph.types.* import java.nio.file.Paths import kotlin.test.Test import kotlin.test.assertEquals