diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3644a9184a..cd6aec2f99 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -22,7 +22,7 @@ jobs: - name: Setup Go uses: actions/setup-go@v4 with: - go-version: "1.20" + go-version: "1.21" - name: Build run: | cd cpg-language-go/src/main/golang @@ -65,7 +65,7 @@ jobs: - name: Setup Go uses: actions/setup-go@v4 with: - go-version: 1.18 + go-version: 1.21 - name: Cache SonarCloud packages uses: actions/cache@v3 with: @@ -126,7 +126,7 @@ jobs: - name: Prepare test and coverage reports if: ${{ always() }} run: | - zip reports.zip **/build/reports/**/** || true + zip -r reports.zip **/build/reports/**/** || true - name: Archive test and coverage reports if: ${{ always() }} uses: actions/upload-artifact@v3 diff --git a/buildSrc/src/main/kotlin/cpg.common-conventions.gradle.kts b/buildSrc/src/main/kotlin/cpg.common-conventions.gradle.kts index dec3f4cc59..721225c614 100644 --- a/buildSrc/src/main/kotlin/cpg.common-conventions.gradle.kts +++ b/buildSrc/src/main/kotlin/cpg.common-conventions.gradle.kts @@ -112,7 +112,11 @@ tasks.withType { // common testing configuration // tasks.test { - useJUnitPlatform() + useJUnitPlatform() { + if (!project.hasProperty("integration")) { + excludeTags("integration") + } + } maxHeapSize = "4048m" } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/ScopeManager.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/ScopeManager.kt index 12129c0d2e..ec6488749c 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/ScopeManager.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/ScopeManager.kt @@ -25,6 +25,7 @@ */ package de.fraunhofer.aisec.cpg +import de.fraunhofer.aisec.cpg.frontends.HasFirstClassFunctions import de.fraunhofer.aisec.cpg.frontends.LanguageFrontend import de.fraunhofer.aisec.cpg.graph.* import de.fraunhofer.aisec.cpg.graph.declarations.* @@ -37,6 +38,7 @@ import de.fraunhofer.aisec.cpg.graph.types.FunctionPointerType import de.fraunhofer.aisec.cpg.graph.types.IncompleteType import de.fraunhofer.aisec.cpg.graph.types.Type import de.fraunhofer.aisec.cpg.helpers.Util +import de.fraunhofer.aisec.cpg.sarif.PhysicalLocation import java.util.* import java.util.function.Predicate import org.slf4j.LoggerFactory @@ -69,6 +71,21 @@ class ScopeManager : ScopeProvider { var currentScope: Scope? = null private set + /** Represents an alias with the name [to] for the particular name [from]. */ + data class Alias(var from: Name, var to: Name) + + /** + * In some languages, we can define aliases for names. An example is renaming package imports in + * Go, e.g., to avoid name conflicts. + * + * In reality, they can probably be defined at different scopes for other languages, but for now + * we only allow it for the current file. + * + * This can potentially be used to replace [addTypedef] at some point, which still relies on the + * existence of a [LanguageFrontend]. + */ + private val aliases = mutableMapOf>() + /** * The language frontend tied to the scope manager. Can be used to implement language specific * scope resolution or lookup. @@ -597,28 +614,45 @@ class ScopeManager : ScopeProvider { * TODO: We should merge this function with [.resolveFunction] */ @JvmOverloads - fun resolveReference(ref: Reference, scope: Scope? = currentScope): ValueDeclaration? { + fun resolveReference(ref: Reference, startScope: Scope? = currentScope): ValueDeclaration? { + // Unfortunately, we still have an issue about duplicate declarations because header files + // are included multiple times, so we need to exclude the C++ frontend (for now). + val language = ref.language + val (scope, name) = + if ( + language?.name?.localName != "CLanguage" && + (language?.name?.localName != "CPPLanguage") + ) { + // For all other languages, we can extract the scope information out of the name and + // start our search at the dedicated scope. + extractScope(ref, startScope) + } else { + Pair(scope, ref.name) + } + + // Try to resolve value declarations according to our criteria return resolve(scope) { - if ( - it.name.lastPartsMatch(ref.name) - ) { // TODO: This place is likely to make things fail - var helper = ref.resolutionHelper - // If the reference seems to point to a function the entire signature is checked - // for equality - if (helper?.type is FunctionPointerType && it is FunctionDeclaration) { - val fptrType = helper.type as FunctionPointerType - // TODO(oxisto): This is the third place where function pointers are - // resolved. WHY? - // TODO(oxisto): Support multiple return values - val returnType = it.returnTypes.firstOrNull() ?: IncompleteType() - if ( + if (it.name.lastPartsMatch(name)) { + val helper = ref.resolutionHelper + return@resolve when { + // If the reference seems to point to a function (using a function pointer) + // the entire signature is checked for equality + helper?.type is FunctionPointerType && it is FunctionDeclaration -> { + val fptrType = helper.type as FunctionPointerType + // TODO(oxisto): Support multiple return values + val returnType = it.returnTypes.firstOrNull() ?: IncompleteType() returnType == fptrType.returnType && it.hasSignature(fptrType.parameters) - ) { - return@resolve true } - } else { - return@resolve it !is FunctionDeclaration + // If our language has first-class functions, we can safely return them as a + // reference + ref.language is HasFirstClassFunctions -> { + true + } + // Otherwise, we are not looking for functions here + else -> { + it !is FunctionDeclaration + } } } @@ -636,23 +670,42 @@ class ScopeManager : ScopeProvider { @JvmOverloads fun resolveFunction( call: CallExpression, - scope: Scope? = currentScope + startScope: Scope? = currentScope ): List { - val s = extractScope(call, scope) + val (scope, name) = extractScope(call, startScope) + + val func = + resolve(scope) { + it.name.lastPartsMatch(name) && it.hasSignature(call.signature) + } - return resolve(s) { it.name.lastPartsMatch(call.name) && it.hasSignature(call.signature) } + return func } - fun extractScope(node: Node, scope: Scope? = currentScope): Scope? { + /** + * This function extracts a possible scope out of a [Name], e.g. if the name is fully qualified. + * This also resolves possible name aliases (e.g. because of imports). It returns a pair of a + * scope (if found) as well as the name, which is possibly adjusted for the aliases. + */ + fun extractScope(node: Node, scope: Scope? = currentScope): Pair { + var name: Name = node.name var s = scope // First, we need to check, whether we have some kind of scoping. if (node.name.parent != null) { // extract the scope name, it is usually a name space, but could probably be something // else as well in other languages - val scopeName = node.name.parent - - // TODO: proper scope selection + var scopeName = node.name.parent + + // We need to check, whether we have an alias for the scope name in this file + val list = aliases[node.location?.artifactLocation] + val alias = list?.firstOrNull { it.to == scopeName }?.from + if (alias != null) { + scopeName = alias + // Reconstruct the original name with the alias, so we can resolve declarations with + // the namespace + name = Name(name.localName, alias) + } // this is a scoped call. we need to explicitly jump to that particular scope val scopes = filterScopes { (it is NameScope && it.name == scopeName) } @@ -669,7 +722,7 @@ class ScopeManager : ScopeProvider { } } - return s + return Pair(s, name) } /** @@ -788,6 +841,12 @@ class ScopeManager : ScopeProvider { .firstOrNull() } + fun addAlias(file: PhysicalLocation.ArtifactLocation, from: Name, to: Name) { + val list = aliases.computeIfAbsent(file) { mutableSetOf() } + + list += Alias(from, to) + } + /** Returns the current scope for the [ScopeProvider] interface. */ override val scope: Scope? get() = currentScope diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TypeManager.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TypeManager.kt index 72bc8aa510..0fc51ab8d4 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TypeManager.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TypeManager.kt @@ -50,12 +50,11 @@ class TypeManager { * Generics) to the ParameterizedType to be able to resolve the Type of the fields, since * ParameterizedTypes are unique to the RecordDeclaration and are not merged. */ - private val recordToTypeParameters = - Collections.synchronizedMap(mutableMapOf>()) - private val templateToTypeParameters = - Collections.synchronizedMap( - mutableMapOf>() - ) + private val recordToTypeParameters: MutableMap> = + ConcurrentHashMap() + private val templateToTypeParameters: + MutableMap> = + ConcurrentHashMap() val firstOrderTypes: MutableSet = ConcurrentHashMap.newKeySet() val secondOrderTypes: MutableSet = ConcurrentHashMap.newKeySet() @@ -288,14 +287,12 @@ internal fun Type.getAncestors(depth: Int): Set { return types } -/** Checks, if this [Type] is either derived from or equals to [superType]. */ +/** + * Checks, if this [Type] is either derived from or equals to [superType]. This is forwarded to the + * [Language] of the [Type] and can be overridden by the individual languages. + */ fun Type.isDerivedFrom(superType: Type): Boolean { - // Retrieve all ancestor types of our type (more concretely of the root type) - val root = this.root - val superTypes = root.ancestors.map { it.type } - - // Check, if super type (or its root) is in the list - return superType.root in superTypes + return this.language?.isDerivedFrom(this, superType) ?: false } /** diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/Language.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/Language.kt index f522474f52..3cb4e01103 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/Language.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/Language.kt @@ -31,6 +31,7 @@ import com.fasterxml.jackson.databind.SerializerProvider import com.fasterxml.jackson.databind.annotation.JsonSerialize import com.fasterxml.jackson.databind.ser.std.StdSerializer import de.fraunhofer.aisec.cpg.TranslationContext +import de.fraunhofer.aisec.cpg.ancestors import de.fraunhofer.aisec.cpg.graph.Name import de.fraunhofer.aisec.cpg.graph.Node import de.fraunhofer.aisec.cpg.graph.statements.expressions.BinaryOperator @@ -199,6 +200,15 @@ abstract class Language> : Node() { return true } + + open fun isDerivedFrom(type: Type, superType: Type): Boolean { + // Retrieve all ancestor types of our type (more concretely of the root type) + val root = type.root + val superTypes = root.ancestors.map { it.type } + + // Check, if super type (or its root) is in the list + return superType.root in superTypes + } } /** diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/LanguageTraits.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/LanguageTraits.kt index cb7278c706..47b427bca3 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/LanguageTraits.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/LanguageTraits.kt @@ -44,16 +44,16 @@ import java.util.regex.Pattern * * Currently, this interface has no methods. However, in the future, this could be used to execute * language/frontend-specific code for the particular trait. This could help to fine-tune the - * [de.fraunhofer.aisec.cpg.passes.CallResolver] for specific languages. + * [CallResolver] for specific languages. */ interface LanguageTrait /** A language trait, that specifies that this language has support for templates or generics. */ interface HasGenerics : LanguageTrait { - /** The char starting the template specific code (e.g. '<') */ + /** The char starting the template specific code (e.g. `<`) */ val startCharacter: Char - /** The char ending the template specific code (e.g. '>') */ + /** The char ending the template specific code (e.g. `>`) */ val endCharacter: Char } @@ -223,3 +223,18 @@ interface HasShortCircuitOperators : LanguageTrait { val operatorCodes: Set get() = conjunctiveOperators.union(disjunctiveOperators) } + +/** + * A language trait, that specifies that this language treats functions "first-class citizens", + * meaning they can be assigned to variables and passed as arguments to other functions. + */ +interface HasFirstClassFunctions + +/** + * A language trait, that specifies that this language has an "anonymous" identifier, used for + * unused parameters or suppressed assignments. + */ +interface HasAnonymousIdentifier { + val anonymousIdentifier: String + get() = "_" +} diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/FunctionDeclaration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/FunctionDeclaration.kt index 49466af8c8..4f87a4fc63 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/FunctionDeclaration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/FunctionDeclaration.kt @@ -117,13 +117,13 @@ open class FunctionDeclaration : ValueDeclaration(), DeclarationHolder { .stream() .sorted(Comparator.comparingInt(ParameterDeclaration::argumentIndex)) .collect(Collectors.toList()) - return if (targetSignature.size < signature.size) { + return if (signature.all { !it.isVariadic } && targetSignature.size < signature.size) { false } else { // signature is a collection of positional arguments, so the order must be preserved for (i in signature.indices) { val declared = signature[i] - if (declared.isVariadic && targetSignature.size >= signature.size) { + if (declared.isVariadic) { // Everything that follows is collected by this param, so the signature is // fulfilled no matter what comes now (potential FIXME: in Java, we could have // overloading with different vararg types, in C++ we can't, as vararg types are diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/IncludeDeclaration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/IncludeDeclaration.kt index 4d9650ee82..db66652ebf 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/IncludeDeclaration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/IncludeDeclaration.kt @@ -34,6 +34,7 @@ import java.util.Objects import org.apache.commons.lang3.builder.ToStringBuilder import org.neo4j.ogm.annotation.Relationship +/** This declaration represents either an include or an import, depending on the language. */ class IncludeDeclaration : Declaration() { @Relationship(value = "INCLUDES", direction = Relationship.Direction.OUTGOING) @AST @@ -43,6 +44,10 @@ class IncludeDeclaration : Declaration() { @AST private val problemEdges: MutableList> = ArrayList() + /** + * This property refers to the file or directory or path. For example, in C this refers to an + * include header file. In Go, this refers to the package path (e.g., github.com/a/b) + */ var filename: String? = null val includes: List by PropertyEdgeDelegate(IncludeDeclaration::includeEdges) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/InitializerListExpression.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/InitializerListExpression.kt index d3c064e6be..0cde0292bf 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/InitializerListExpression.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/InitializerListExpression.kt @@ -69,7 +69,14 @@ class InitializerListExpression : Expression(), ArgumentHolder, HasType.TypeObse } override fun replaceArgument(old: Expression, new: Expression): Boolean { - // Not supported, too complex + val idx = initializerEdges.indexOfFirst { it.end == old } + if (idx != -1) { + old.unregisterTypeObserver(this) + initializerEdges[idx].end = new + new.registerTypeObserver(this) + return true + } + return false } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/KeyValueExpression.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/KeyValueExpression.kt index 7ca1b0d42d..c5bda8735e 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/KeyValueExpression.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/KeyValueExpression.kt @@ -26,6 +26,7 @@ package de.fraunhofer.aisec.cpg.graph.statements.expressions import de.fraunhofer.aisec.cpg.graph.AST +import de.fraunhofer.aisec.cpg.graph.ArgumentHolder import java.util.* /** @@ -35,7 +36,7 @@ import java.util.* * Most often used in combination with an [InitializerListExpression] to represent the creation of * an array. */ -class KeyValueExpression : Expression() { +class KeyValueExpression : Expression(), ArgumentHolder { /** * The key of this pair. It is usually a literal, but some languages even allow references to @@ -46,6 +47,26 @@ class KeyValueExpression : Expression() { /** The value of this pair. It can be any expression */ @AST var value: Expression? = null + override fun addArgument(expression: Expression) { + if (key == null) { + key = expression + } else if (value == null) { + value = expression + } + } + + override fun replaceArgument(old: Expression, new: Expression): Boolean { + if (key == old) { + key = new + return true + } else if (value == old) { + value = new + return true + } + + return false + } + override fun equals(other: Any?): Boolean { if (this === other) return true if (other !is KeyValueExpression) return false diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/MemberExpression.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/MemberExpression.kt index 3c599ee29d..409a5269b2 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/MemberExpression.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/MemberExpression.kt @@ -26,6 +26,7 @@ package de.fraunhofer.aisec.cpg.graph.statements.expressions import de.fraunhofer.aisec.cpg.graph.AST +import de.fraunhofer.aisec.cpg.graph.ArgumentHolder import de.fraunhofer.aisec.cpg.graph.HasBase import de.fraunhofer.aisec.cpg.graph.declarations.RecordDeclaration import de.fraunhofer.aisec.cpg.graph.fqn @@ -39,7 +40,7 @@ import org.apache.commons.lang3.builder.ToStringBuilder * use-case is access of a member function (method) as part of the [MemberCallExpression.callee] * property of a [MemberCallExpression]. */ -class MemberExpression : Reference(), HasBase { +class MemberExpression : Reference(), ArgumentHolder, HasBase { @AST override var base: Expression = ProblemExpression("could not parse base expression") set(value) { @@ -58,6 +59,19 @@ class MemberExpression : Reference(), HasBase { .toString() } + override fun addArgument(expression: Expression) { + this.base = expression + } + + override fun replaceArgument(old: Expression, new: Expression): Boolean { + if (old == base) { + base = new + return true + } + + return false + } + override fun equals(other: Any?): Boolean { if (this === other) return true if (other !is MemberExpression) return false diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/Reference.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/Reference.kt index e7c06039fb..27f25e11e3 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/Reference.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/Reference.kt @@ -136,7 +136,11 @@ open class Reference : Expression(), HasType.TypeObserver { if (other !is Reference) { return false } - return super.equals(other) && refersTo == other.refersTo + return super.equals(other) + } + + override fun hashCode(): Int { + return super.hashCode() } override fun addPrevDFG(prev: Node, properties: MutableMap) { @@ -149,6 +153,4 @@ open class Reference : Expression(), HasType.TypeObserver { prev.registerTypeObserver(this) } } - - override fun hashCode(): Int = Objects.hash(super.hashCode(), refersTo) } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/UnaryOperator.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/UnaryOperator.kt index 6fde30092f..d90d296e0e 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/UnaryOperator.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/UnaryOperator.kt @@ -27,13 +27,14 @@ package de.fraunhofer.aisec.cpg.graph.statements.expressions import de.fraunhofer.aisec.cpg.graph.AST import de.fraunhofer.aisec.cpg.graph.AccessValues +import de.fraunhofer.aisec.cpg.graph.ArgumentHolder import de.fraunhofer.aisec.cpg.graph.pointer import de.fraunhofer.aisec.cpg.graph.types.HasType import de.fraunhofer.aisec.cpg.graph.types.Type import org.apache.commons.lang3.builder.ToStringBuilder /** A unary operator expression, involving one expression and an operator, such as `a++`. */ -class UnaryOperator : Expression(), HasType.TypeObserver { +class UnaryOperator : Expression(), ArgumentHolder, HasType.TypeObserver { /** The expression on which the operation is applied. */ @AST var input: Expression = ProblemExpression("could not parse input") @@ -112,6 +113,19 @@ class UnaryOperator : Expression(), HasType.TypeObserver { ) } + override fun addArgument(expression: Expression) { + this.input = expression + } + + override fun replaceArgument(old: Expression, new: Expression): Boolean { + if (this.input == old) { + this.input = new + return true + } + + return false + } + override fun equals(other: Any?): Boolean { if (this === other) { return true diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/CallResolver.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/CallResolver.kt index b22aa025bd..f62186db48 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/CallResolver.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/CallResolver.kt @@ -200,7 +200,7 @@ open class CallResolver(ctx: TranslationContext) : SymbolResolverPass(ctx) { // the TU. It is also a little bit redundant, since ScopeManager.resolveFunction // (which gets called before) already extracts the scope, but this information // gets lost. - val scope = scopeManager.extractScope(call, scopeManager.globalScope) + val (scope, _) = scopeManager.extractScope(call, scopeManager.globalScope) // We have two possible start points, a namespace declaration or a translation // unit. Nothing else is allowed (fow now) @@ -321,6 +321,11 @@ open class CallResolver(ctx: TranslationContext) : SymbolResolverPass(ctx) { curClass: RecordDeclaration?, call: CallExpression ): List { + // We need to resolve the base calls first. This might be done duplicate now + if (callee is MemberExpression && callee.base is CallExpression) { + handleCallExpression(curClass, callee.base as CallExpression) + } + // We need to adjust certain types of the base in case of a super call and we delegate this. // If that is successful, we can continue with regular resolving if ( diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/ControlFlowSensitiveDFGPass.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/ControlFlowSensitiveDFGPass.kt index 72317c4760..656ecbed93 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/ControlFlowSensitiveDFGPass.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/ControlFlowSensitiveDFGPass.kt @@ -200,11 +200,19 @@ open class ControlFlowSensitiveDFGPass(ctx: TranslationContext) : TranslationUni // the "normal" case won't work. We handle this case separately here... // This is what we write to the declaration val iterable = currentNode.iterable as? Expression - val writtenTo = - when (currentNode.variable) { - is DeclarationStatement -> - (currentNode.variable as DeclarationStatement).singleDeclaration + when (val variable = currentNode.variable) { + is DeclarationStatement -> { + if (variable.isSingleDeclaration()) { + variable.singleDeclaration + } else if (variable.variables.size == 2) { + // If there are two variables, we just blindly assume that the order is + // (key, value), so we return the second one + variable.declarations[1] + } else { + null + } + } else -> currentNode.variable } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/EvaluationOrderGraphPass.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/EvaluationOrderGraphPass.kt index 852cb314cc..fa4f4c6ec9 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/EvaluationOrderGraphPass.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/EvaluationOrderGraphPass.kt @@ -127,6 +127,7 @@ open class EvaluationOrderGraphPass(ctx: TranslationContext) : TranslationUnitPa map[DoStatement::class.java] = { handleDoStatement(it as DoStatement) } map[ForStatement::class.java] = { handleForStatement(it as ForStatement) } map[ForEachStatement::class.java] = { handleForEachStatement(it as ForEachStatement) } + map[TypeExpression::class.java] = { handleTypeExpression(it as TypeExpression) } map[TryStatement::class.java] = { handleTryStatement(it as TryStatement) } map[ContinueStatement::class.java] = { handleContinueStatement(it as ContinueStatement) } map[DeleteExpression::class.java] = { handleDeleteExpression(it as DeleteExpression) } @@ -244,7 +245,7 @@ open class EvaluationOrderGraphPass(ctx: TranslationContext) : TranslationUnitPa pushToEOG(node) } - protected fun handleRecordDeclaration(node: RecordDeclaration) { + protected open fun handleRecordDeclaration(node: RecordDeclaration) { scopeManager.enterScope(node) handleStatementHolder(node) currentPredecessors.clear() @@ -584,6 +585,10 @@ open class EvaluationOrderGraphPass(ctx: TranslationContext) : TranslationUnitPa pushToEOG(node) } + protected fun handleTypeExpression(node: TypeExpression) { + pushToEOG(node) + } + protected fun handleTryStatement(node: TryStatement) { scopeManager.enterScope(node) val tryScope = scopeManager.currentScope as TryScope? diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/TypeResolver.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/TypeResolver.kt index 4891b229c0..3def0992f5 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/TypeResolver.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/TypeResolver.kt @@ -29,7 +29,8 @@ import de.fraunhofer.aisec.cpg.TranslationContext import de.fraunhofer.aisec.cpg.graph.Component import de.fraunhofer.aisec.cpg.graph.Node import de.fraunhofer.aisec.cpg.graph.declarations.RecordDeclaration -import de.fraunhofer.aisec.cpg.graph.types.* +import de.fraunhofer.aisec.cpg.graph.types.ObjectType +import de.fraunhofer.aisec.cpg.graph.types.Type import de.fraunhofer.aisec.cpg.processing.IVisitor import de.fraunhofer.aisec.cpg.processing.strategy.Strategy diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/VariableUsageResolver.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/VariableUsageResolver.kt index 80c6b2a30a..edb95d011e 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/VariableUsageResolver.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/VariableUsageResolver.kt @@ -26,6 +26,7 @@ package de.fraunhofer.aisec.cpg.passes import de.fraunhofer.aisec.cpg.TranslationContext +import de.fraunhofer.aisec.cpg.frontends.HasAnonymousIdentifier import de.fraunhofer.aisec.cpg.frontends.HasStructs import de.fraunhofer.aisec.cpg.frontends.HasSuperClasses import de.fraunhofer.aisec.cpg.graph.* @@ -114,6 +115,14 @@ open class VariableUsageResolver(ctx: TranslationContext) : SymbolResolverPass(c if (current !is Reference || current is MemberExpression) return + // We can safely ignore references to the anonymous identifier, e.g. _ in Go. + if ( + language is HasAnonymousIdentifier && + current.name.localName == language.anonymousIdentifier + ) { + return + } + // For now, we need to ignore reference expressions that are directly embedded into call // expressions, because they are the "callee" property. In the future, we will use this // property to actually resolve the function call. However, there is a special case that @@ -299,6 +308,14 @@ open class VariableUsageResolver(ctx: TranslationContext) : SymbolResolverPass(c } protected fun resolveBase(reference: Reference): Declaration? { + // We need to check, whether we first need to resolve our own base + if (reference is MemberExpression) { + val base = reference.base + if (base is Reference && base.refersTo == null) { + base.refersTo = resolveBase(base) + } + } + val declaration = scopeManager.resolveReference(reference) if (declaration != null) { return declaration diff --git a/cpg-core/src/testFixtures/kotlin/de/fraunhofer/aisec/cpg/TestUtils.kt b/cpg-core/src/testFixtures/kotlin/de/fraunhofer/aisec/cpg/TestUtils.kt index 9ab21d3552..1f5ca2cab3 100644 --- a/cpg-core/src/testFixtures/kotlin/de/fraunhofer/aisec/cpg/TestUtils.kt +++ b/cpg-core/src/testFixtures/kotlin/de/fraunhofer/aisec/cpg/TestUtils.kt @@ -26,7 +26,7 @@ package de.fraunhofer.aisec.cpg import de.fraunhofer.aisec.cpg.frontends.CompilationDatabase -import de.fraunhofer.aisec.cpg.graph.Node +import de.fraunhofer.aisec.cpg.graph.* import de.fraunhofer.aisec.cpg.graph.declarations.Declaration import de.fraunhofer.aisec.cpg.graph.declarations.FunctionDeclaration import de.fraunhofer.aisec.cpg.graph.declarations.TranslationUnitDeclaration @@ -154,7 +154,9 @@ object TestUtils { ): List { val config = builder.build() val analyzer = TranslationManager.builder().config(config).build() - return analyzer.analyze().get().translationUnits + val result = analyzer.analyze().get() + + return result.components["application"]?.translationUnits ?: listOf() } @JvmOverloads diff --git a/cpg-language-go/build.gradle.kts b/cpg-language-go/build.gradle.kts index fc19473bca..984f246c5d 100644 --- a/cpg-language-go/build.gradle.kts +++ b/cpg-language-go/build.gradle.kts @@ -41,6 +41,7 @@ publishing { dependencies { implementation("net.java.dev.jna:jna:5.13.0") + testImplementation(project(":cpg-analysis")) } if (!project.hasProperty("skipGoBuild")) { diff --git a/cpg-language-go/src/main/golang/lib/cpg/main.go b/cpg-language-go/src/main/golang/lib/cpg/main.go index f276a47a1e..351f9e0450 100644 --- a/cpg-language-go/src/main/golang/lib/cpg/main.go +++ b/cpg-language-go/src/main/golang/lib/cpg/main.go @@ -60,10 +60,10 @@ func NewFileSet() unsafe.Pointer { } //export goParserParseFile -func goParserParseFile(fset unsafe.Pointer, path *C.char, src *C.char) unsafe.Pointer { +func goParserParseFile(fset unsafe.Pointer, path *C.char) unsafe.Pointer { f, err := parser.ParseFile( pointer.Restore(fset).(*token.FileSet), - C.GoString(path), C.GoString(src), parser.ParseComments) + C.GoString(path), nil, parser.ParseComments|parser.SkipObjectResolution) if err != nil { panic(err) } @@ -175,9 +175,10 @@ func GetCommentMapNodeComment(ptr1 unsafe.Pointer, ptr2 unsafe.Pointer) *C.char } for _, c := range comments { - text := strings.TrimRight(c.Text(), "\n") - comment += text + comment += c.Text() } + // Remove last \n + comment = strings.TrimRight(comment, "\n") return C.CString(comment) } @@ -286,6 +287,12 @@ func GetCallExprArg(ptr unsafe.Pointer, i int) unsafe.Pointer { }) } +//export GetEllipsisElt +func GetEllipsisElt(ptr unsafe.Pointer) unsafe.Pointer { + expr := restore[*ast.Ellipsis](ptr) + return save(expr.Elt) +} + //export GetForStmtInit func GetForStmtInit(ptr unsafe.Pointer) unsafe.Pointer { stmt := restore[*ast.ForStmt](ptr) @@ -466,6 +473,26 @@ func GetIndexExprIndex(ptr unsafe.Pointer) unsafe.Pointer { return save(expr.Index) } +//export GetIndexListExprX +func GetIndexListExprX(ptr unsafe.Pointer) unsafe.Pointer { + expr := restore[*ast.IndexListExpr](ptr) + return save(expr.X) +} + +//export GetNumIndexListExprIndices +func GetNumIndexListExprIndices(ptr unsafe.Pointer) C.int { + return num[*ast.IndexListExpr](ptr, func(t *ast.IndexListExpr) []ast.Expr { + return t.Indices + }) +} + +//export GetIndexListExprIndex +func GetIndexListExprIndex(ptr unsafe.Pointer, i int) unsafe.Pointer { + return item[*ast.IndexListExpr](ptr, i, func(t *ast.IndexListExpr) []ast.Expr { + return t.Indices + }) +} + //export GetKeyValueExprKey func GetKeyValueExprKey(ptr unsafe.Pointer) unsafe.Pointer { kv := restore[*ast.KeyValueExpr](ptr) @@ -478,6 +505,12 @@ func GetKeyValueExprValue(ptr unsafe.Pointer) unsafe.Pointer { return save(kv.Value) } +//export GetParenExprX +func GetParenExprX(ptr unsafe.Pointer) unsafe.Pointer { + p := restore[*ast.ParenExpr](ptr) + return save(p.X) +} + //export GetSelectorExprX func GetSelectorExprX(ptr unsafe.Pointer) unsafe.Pointer { sel := restore[*ast.SelectorExpr](ptr) @@ -643,6 +676,12 @@ func GetGenDeclSpec(ptr unsafe.Pointer, i int) unsafe.Pointer { }) } +//export GetGenDeclTok +func GetGenDeclTok(ptr unsafe.Pointer) int { + decl := restore[*ast.GenDecl](ptr) + return int(decl.Tok) +} + //export GetInterfaceTypeMethods func GetInterfaceTypeMethods(ptr unsafe.Pointer) unsafe.Pointer { i := restore[*ast.InterfaceType](ptr) @@ -739,6 +778,12 @@ func GetTypeSpecName(ptr unsafe.Pointer) unsafe.Pointer { return save(spec.Name) } +//export GetTypeSpecAssign +func GetTypeSpecAssign(ptr unsafe.Pointer) int { + spec := restore[*ast.TypeSpec](ptr) + return int(spec.Assign) +} + //export GetTypeSpecType func GetTypeSpecType(ptr unsafe.Pointer) unsafe.Pointer { spec := restore[*ast.TypeSpec](ptr) @@ -847,6 +892,18 @@ func GetReturnStmtResult(ptr unsafe.Pointer, i int) unsafe.Pointer { }) } +//export GetSendStmtChan +func GetSendStmtChan(ptr unsafe.Pointer) unsafe.Pointer { + stmt := restore[*ast.SendStmt](ptr) + return save(stmt.Chan) +} + +//export GetSendStmtValue +func GetSendStmtValue(ptr unsafe.Pointer) unsafe.Pointer { + stmt := restore[*ast.SendStmt](ptr) + return save(stmt.Value) +} + //export GetSwitchStmtInit func GetSwitchStmtInit(ptr unsafe.Pointer) unsafe.Pointer { stmt := restore[*ast.SwitchStmt](ptr) @@ -865,6 +922,24 @@ func GetSwitchStmtBody(ptr unsafe.Pointer) unsafe.Pointer { return save(stmt.Body) } +//export GetTypeSwitchStmtInit +func GetTypeSwitchStmtInit(ptr unsafe.Pointer) unsafe.Pointer { + stmt := restore[*ast.TypeSwitchStmt](ptr) + return save(stmt.Init) +} + +//export GetTypeSwitchStmtAssign +func GetTypeSwitchStmtAssign(ptr unsafe.Pointer) unsafe.Pointer { + stmt := restore[*ast.TypeSwitchStmt](ptr) + return save(stmt.Assign) +} + +//export GetTypeSwitchStmtBody +func GetTypeSwitchStmtBody(ptr unsafe.Pointer) unsafe.Pointer { + stmt := restore[*ast.TypeSwitchStmt](ptr) + return save(stmt.Body) +} + func restore[T any](ptr unsafe.Pointer) T { return pointer.Restore(ptr).(T) } diff --git a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/BuildConstraints.kt b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/BuildConstraints.kt new file mode 100644 index 0000000000..ad0ed83f77 --- /dev/null +++ b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/BuildConstraints.kt @@ -0,0 +1,177 @@ +/* + * Copyright (c) 2023, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.frontends.golang + +import java.util.StringTokenizer + +/** + * Go allows to specify so-called + * [build constraints](https://pkg.go.dev/cmd/go#hdr-Build_constraints) on files, to specify + * conditions on which they should be built (or not). + * + * For example, the following build constraint specifies that the file is either built on linux and + * darwin arm-based machines: + * ``` + * //go:build (linux && arm64) || (darwin && arm64) + * ``` + * + * These build constraints consist of several expressions (tags) that can be combined with `||` and + * `&&` as well as negated with `!`. + * + * This interface serves as the bases for all these expressions and contains a companion object to + * parse such an expression (without the `//go:build` prefix) using [fromString]. + */ +fun interface BuildConstraintExpression { + /** + * Evaluates whether the current expression satisfies the set of [tags]. Tags can be for example + * the target operating system (e.g., `linux`), the target architecture (e.g. `arm64`), the Go + * version (e.g. `go1.20`) or other custom tags supplied to the build process. + */ + fun evaluate(tags: Set): Boolean + + companion object { + /** + * This function can be used to translate a string-based build constraint (without the + * `//go:build` prefix) into a [BuildConstraintExpression]. + */ + fun fromString(str: String): BuildConstraintExpression? { + val tokenizer = StringTokenizer(str.replace(" ", ""), "!&|()", true) + + return tokenizer.readExpression() + } + + private fun StringTokenizer.readExpression(): BuildConstraintExpression? { + if (!this.hasMoreTokens()) { + return null + } + + when (val token = nextToken()) { + !in listOf("!", "(", ")", "&", "|") -> { + // A tag + val tag = BuildConstraintTag(token) + + // We need to check, if there is more + if (hasMoreTokens()) { + // If there is more, there needs to be an operator code + val op = readOperatorCode() ?: return null + + // There needs to be another expression + val rhs = readExpression() ?: return null + + return BuildConstraintBinaryExpression(op, tag, rhs) + } else { + return tag + } + } + "!" -> { + // A unary expression + val expr = readExpression() ?: return null + return BuildConstraintUnaryExpression("!", expr) + } + "(" -> { + val inner = this.nextToken(")") + val expr = fromString(inner) ?: return null + + return BuildConstraintParenthesis(expr) + } + else -> return null + } + } + + private fun StringTokenizer.readOperatorCode(): String? { + if (!this.hasMoreTokens()) { + return null + } + + val char1 = this.nextToken() + if (char1 != "|" && char1 != "&") { + return null + } + + if (!this.hasMoreTokens()) { + return null + } + + val char2 = this.nextToken() + if (char2 != char1) { + return null + } + + return char1 + char2 + } + } +} + +/** + * A unary build constraint expression. Currently, only `!` as [operatorCode] for a negation is + * supported. + */ +class BuildConstraintUnaryExpression( + val operatorCode: String, + val expr: BuildConstraintExpression +) : BuildConstraintExpression { + override fun evaluate(tags: Set): Boolean { + if (operatorCode == "!") { + return !expr.evaluate(tags) + } else { + TODO() + } + } +} + +/** + * A binary build constraint expression. Currently, only `&&` and `||` as [operatorCode] are + * supported. + */ +class BuildConstraintBinaryExpression( + val operatorCode: String, + val lhs: BuildConstraintExpression, + val rhs: BuildConstraintExpression +) : BuildConstraintExpression { + override fun evaluate(tags: Set): Boolean { + return when (operatorCode) { + "&&" -> lhs.evaluate(tags) && rhs.evaluate(tags) + "||" -> lhs.evaluate(tags) || rhs.evaluate(tags) + else -> TODO() + } + } +} + +/** + * A simple parenthesis around a build constraint, which can be used to logically group expressions. + */ +class BuildConstraintParenthesis(val expr: BuildConstraintExpression) : + BuildConstraintExpression by expr + +/** + * The usage of a tag as a build constraint expression. This is the simplest form of expression and + * will return true if the [tag] is contained in the list of available tags. + */ +class BuildConstraintTag(val tag: String) : BuildConstraintExpression { + override fun evaluate(tags: Set): Boolean { + return tag in tags + } +} diff --git a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/DeclarationHandler.kt b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/DeclarationHandler.kt index 5aa878b488..28d450bd65 100644 --- a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/DeclarationHandler.kt +++ b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/DeclarationHandler.kt @@ -25,8 +25,6 @@ */ package de.fraunhofer.aisec.cpg.frontends.golang -import de.fraunhofer.aisec.cpg.frontends.Handler -import de.fraunhofer.aisec.cpg.frontends.HandlerInterface import de.fraunhofer.aisec.cpg.graph.* import de.fraunhofer.aisec.cpg.graph.declarations.* import de.fraunhofer.aisec.cpg.graph.statements.ReturnStatement @@ -35,36 +33,31 @@ import de.fraunhofer.aisec.cpg.graph.types.Type import de.fraunhofer.aisec.cpg.graph.types.UnknownType class DeclarationHandler(frontend: GoLanguageFrontend) : - Handler( - ::ProblemDeclaration, - frontend - ) { - - init { - map[GoStandardLibrary.Ast.FuncDecl::class.java] = HandlerInterface { - handleFuncDecl(it as GoStandardLibrary.Ast.FuncDecl) - } - map[GoStandardLibrary.Ast.GenDecl::class.java] = HandlerInterface { - handleGenDecl(it as GoStandardLibrary.Ast.GenDecl) + GoHandler(::ProblemDeclaration, frontend) { + + override fun handleNode(node: GoStandardLibrary.Ast.Decl): Declaration? { + return when (node) { + is GoStandardLibrary.Ast.FuncDecl -> handleFuncDecl(node) + is GoStandardLibrary.Ast.GenDecl -> handleGenDecl(node) + else -> { + return handleNotSupported(node, node.goType) + } } - map.put(GoStandardLibrary.Ast.Decl::class.java, ::handleNode) - } - - private fun handleNode(decl: GoStandardLibrary.Ast.Decl): Declaration { - val message = "Not parsing declaration of type ${decl.goType} yet" - log.error(message) - - return newProblemDeclaration(message) } private fun handleFuncDecl(funcDecl: GoStandardLibrary.Ast.FuncDecl): FunctionDeclaration { val recv = funcDecl.recv val func = if (recv != null) { - val method = newMethodDeclaration(funcDecl.name.name, rawNode = funcDecl) val recvField = recv.list.firstOrNull() val recordType = recvField?.type?.let { frontend.typeOf(it) } ?: unknownType() + val method = + newMethodDeclaration( + Name(funcDecl.name.name, recordType.root.name), + rawNode = funcDecl + ) + // The name of the Go receiver is optional. In fact, if the name is not // specified we probably do not need any receiver variable at all, // because the syntax is only there to ensure that this method is part @@ -79,7 +72,7 @@ class DeclarationHandler(frontend: GoLanguageFrontend) : } if (recordType !is UnknownType) { - val recordName = recordType.name + val recordName = recordType.root.name // TODO: this will only find methods within the current translation unit. // this is a limitation that we have for C++ as well @@ -93,9 +86,14 @@ class DeclarationHandler(frontend: GoLanguageFrontend) : // marked as AST and in Go a method is not part of the struct's AST but is // declared outside. In the future, we need to differentiate between just the // associated members of the class and the pure AST nodes declared in the - // struct - // itself - record?.addMethod(method) + // struct itself + if (record != null) { + method.recordDeclaration = record + record.addMethod(method) + + // Enter scope of record + frontend.scopeManager.enterScope(record) + } } method } else { @@ -143,69 +141,93 @@ class DeclarationHandler(frontend: GoLanguageFrontend) : // Parse parameters for (param in funcDecl.type.params.list) { - var name = "" - - // Somehow parameters end up having no name sometimes, have not fully understood why. - if (param.names.isNotEmpty()) { - name = param.names[0].name - - // If the name is an underscore, it means that the parameter is - // unnamed. In order to avoid confusing and some compatibility with - // other languages, we are just setting the name to an empty string - // in this case. - if (name == "_") { - name = "" - } + // We need to differentiate between three cases: + // - an empty list of names, which means that the parameter is unnamed; and we also give + // it an empty name + // - a single entry in the list of names, which is one regular parameter + // - multiple entries in the list of names, which specifies multiple parameters with the + // same type + // + // We can treat the last two cases together, by just gathering all the names and + // creating one param per name with the same type + val names = mutableListOf() + if (param.names.isEmpty()) { + names += "" } else { - log.warn("Some param has no name, which is a bit weird: $param") + names += + param.names.map { + // If the name is an underscore, it means that the parameter is + // unused (but not unnamed). + // + // But, but order to avoid confusion in resolving and to add + // some compatibility with other languages, we are just setting + // the name to an empty string in this case as well. + val name = it.name + if (name == "_") { + return@map "" + } + name + } } - // Check for varargs. In this case we want to parse the element type - // (and make it an array afterwards) - var variadic = false - val type = - if (param.type is GoStandardLibrary.Ast.Ellipsis) { - variadic = true - frontend.typeOf((param.type as GoStandardLibrary.Ast.Ellipsis).elt).array() - } else { - frontend.typeOf(param.type) - } - - val p = newParameterDeclaration(name, type, variadic, rawNode = param) + // Create one param variable per name + for (name in names) { + // Check for varargs. In this case we want to parse the element type + // (and make it an array afterwards) + val (type, variadic) = frontend.fieldTypeOf(param.type) - frontend.scopeManager.addDeclaration(p) + val p = newParameterDeclaration(name, type, variadic, rawNode = param) - frontend.setComment(p, param) + frontend.scopeManager.addDeclaration(p) + frontend.setComment(p, param) + } } - // Check, if the last statement is a return statement, otherwise we insert an implicit one - val body = frontend.statementHandler.handle(funcDecl.body) - if (body is Block) { - val last = body.statements.lastOrNull() - if (last !is ReturnStatement) { - val ret = newReturnStatement() - ret.isImplicit = true - body += ret + // Only parse function body in non-dependencies + if (!frontend.isDependency) { + // Check, if the last statement is a return statement, otherwise we insert an implicit + // one + val body = funcDecl.body?.let { frontend.statementHandler.handle(it) } + if (body is Block) { + val last = body.statements.lastOrNull() + if (last !is ReturnStatement) { + val ret = newReturnStatement() + ret.isImplicit = true + body += ret + } } + func.body = body } - func.body = body - frontend.scopeManager.leaveScope(func) + // Leave scope of record, if applicable + (func as? MethodDeclaration)?.recordDeclaration?.let { + frontend.scopeManager.leaveScope(it) + } + return func } private fun handleGenDecl(genDecl: GoStandardLibrary.Ast.GenDecl): DeclarationSequence { + // Reset the iota value. We need to start with -1 because we immediately increment in + // handleValueSpec + frontend.declCtx.iotaValue = -1 + // Set ourselves as the current gendecl + frontend.declCtx.currentDecl = genDecl + // Reset the initializers + frontend.declCtx.constInitializers.clear() + val sequence = DeclarationSequence() for (spec in genDecl.specs) { - frontend.specificationHandler.handle(spec)?.let { - sequence += it + val declaration = frontend.specificationHandler.handle(spec) + if (declaration != null) { + sequence += declaration // Go associates the comment to the genDecl, so we need to explicitly launch // setComment here. - frontend.setComment(it, genDecl) + frontend.setComment(declaration, genDecl) } } diff --git a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/ExpressionHandler.kt b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/ExpressionHandler.kt index ee5c702d98..6055a14c7a 100644 --- a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/ExpressionHandler.kt +++ b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/ExpressionHandler.kt @@ -29,35 +29,41 @@ import de.fraunhofer.aisec.cpg.frontends.golang.GoStandardLibrary.Ast.BasicLit.K import de.fraunhofer.aisec.cpg.graph.* import de.fraunhofer.aisec.cpg.graph.declarations.FunctionDeclaration import de.fraunhofer.aisec.cpg.graph.statements.expressions.* +import de.fraunhofer.aisec.cpg.graph.types.ObjectType import de.fraunhofer.aisec.cpg.graph.types.PointerType import de.fraunhofer.aisec.cpg.graph.types.Type +import java.math.BigInteger class ExpressionHandler(frontend: GoLanguageFrontend) : GoHandler(::ProblemExpression, frontend) { - override fun handleNode(expr: GoStandardLibrary.Ast.Expr): Expression { - return when (expr) { - is GoStandardLibrary.Ast.BasicLit -> handleBasicLit(expr) - is GoStandardLibrary.Ast.BinaryExpr -> handleBinaryExpr(expr) - is GoStandardLibrary.Ast.CompositeLit -> handleCompositeLit(expr) - is GoStandardLibrary.Ast.FuncLit -> handleFuncLit(expr) - is GoStandardLibrary.Ast.Ident -> handleIdent(expr) - is GoStandardLibrary.Ast.IndexExpr -> handleIndexExpr(expr) - is GoStandardLibrary.Ast.CallExpr -> handleCallExpr(expr) - is GoStandardLibrary.Ast.KeyValueExpr -> handleKeyValueExpr(expr) - is GoStandardLibrary.Ast.SelectorExpr -> handleSelectorExpr(expr) - is GoStandardLibrary.Ast.SliceExpr -> handleSliceExpr(expr) - is GoStandardLibrary.Ast.TypeAssertExpr -> handleTypeAssertExpr(expr) - is GoStandardLibrary.Ast.UnaryExpr -> handleUnaryExpr(expr) + override fun handleNode(node: GoStandardLibrary.Ast.Expr): Expression { + return when (node) { + is GoStandardLibrary.Ast.BasicLit -> handleBasicLit(node) + is GoStandardLibrary.Ast.BinaryExpr -> handleBinaryExpr(node) + is GoStandardLibrary.Ast.CompositeLit -> handleCompositeLit(node) + is GoStandardLibrary.Ast.FuncLit -> handleFuncLit(node) + is GoStandardLibrary.Ast.Ident -> handleIdent(node) + is GoStandardLibrary.Ast.IndexExpr -> handleIndexExpr(node) + is GoStandardLibrary.Ast.CallExpr -> handleCallExpr(node) + is GoStandardLibrary.Ast.KeyValueExpr -> handleKeyValueExpr(node) + is GoStandardLibrary.Ast.ParenExpr -> { + return handle(node.x) + } + is GoStandardLibrary.Ast.SelectorExpr -> handleSelectorExpr(node) + is GoStandardLibrary.Ast.SliceExpr -> handleSliceExpr(node) + is GoStandardLibrary.Ast.StarExpr -> handleStarExpr(node) + is GoStandardLibrary.Ast.TypeAssertExpr -> handleTypeAssertExpr(node) + is GoStandardLibrary.Ast.UnaryExpr -> handleUnaryExpr(node) else -> { - return handleNotSupported(expr, expr.goType) + return handleNotSupported(node, node.goType) } } } private fun handleBasicLit(basicLit: GoStandardLibrary.Ast.BasicLit): Literal<*> { - val rawValue = basicLit.value - val value: Any? + var rawValue = basicLit.value + var value: Any? val type: Type when (basicLit.kind) { STRING -> { @@ -69,7 +75,31 @@ class ExpressionHandler(frontend: GoLanguageFrontend) : type = primitiveType("string") } INT -> { - value = rawValue.toInt() + // Get rid of all underscores + rawValue = rawValue.replace("_", "") + val prefix = rawValue.substring(0, 2.coerceAtMost(rawValue.length)) + val postfix = rawValue.substring(2.coerceAtMost(rawValue.length), rawValue.length) + + value = + when (prefix) { + "0x" -> BigInteger(postfix, 16) + "0o" -> BigInteger(postfix, 10) + "0b" -> BigInteger(postfix, 2) + else -> BigInteger(rawValue, 10) + } + + value = + when { + value > BigInteger.valueOf(Long.MAX_VALUE) -> { + value + } + value.toLong() > Int.MAX_VALUE -> { + value.toLong() + } + else -> { + value.toInt() + } + } type = primitiveType("int") } FLOAT -> { @@ -94,16 +124,26 @@ class ExpressionHandler(frontend: GoLanguageFrontend) : private fun handleBinaryExpr(binaryExpr: GoStandardLibrary.Ast.BinaryExpr): BinaryOperator { val binOp = newBinaryOperator(binaryExpr.opString, rawNode = binaryExpr) - binOp.lhs = handle(binaryExpr.x) ?: newProblemExpression("missing LHS") - binOp.rhs = handle(binaryExpr.y) ?: newProblemExpression("missing RHS") + binOp.lhs = handle(binaryExpr.x) + binOp.rhs = handle(binaryExpr.y) return binOp } private fun handleIdent(ident: GoStandardLibrary.Ast.Ident): Expression { - // Check, if this is 'nil', because then we handle it as a literal in the graph - if (ident.name == "nil") { - val literal = newLiteral(null, rawNode = ident) + val builtinLiterals = + mapOf( + "nil" to Pair(unknownType(), null), + "true" to Pair(primitiveType("bool"), true), + "false" to Pair(primitiveType("bool"), false), + "iota" to Pair(primitiveType("int"), frontend.declCtx.iotaValue) + ) + + // Check, if this is one of the builtinLiterals and handle them as a literal + val literalPair = builtinLiterals[ident.name] + if (literalPair != null) { + val (type, value) = literalPair + val literal = newLiteral(value, type, rawNode = ident) literal.name = parseName(ident.name) return literal @@ -132,7 +172,7 @@ class ExpressionHandler(frontend: GoLanguageFrontend) : private fun handleCallExpr(callExpr: GoStandardLibrary.Ast.CallExpr): Expression { // In Go, regular cast expressions (not type asserts are modelled as calls). // In this case, the Fun contains a type expression. - when (callExpr.`fun`) { + when (val unwrapped = unwrap(callExpr.`fun`)) { is GoStandardLibrary.Ast.ArrayType, is GoStandardLibrary.Ast.ChanType, is GoStandardLibrary.Ast.FuncType, @@ -140,22 +180,35 @@ class ExpressionHandler(frontend: GoLanguageFrontend) : is GoStandardLibrary.Ast.StructType, is GoStandardLibrary.Ast.MapType, -> { val cast = newCastExpression(rawNode = callExpr) - cast.castType = frontend.typeOf(callExpr.`fun`) + cast.castType = frontend.typeOf(unwrapped) if (callExpr.args.isNotEmpty()) { - frontend.expressionHandler.handle(callExpr.args[0])?.let { - cast.expression = it - } + cast.expression = frontend.expressionHandler.handle(callExpr.args[0]) } return cast } } - // Parse the Fun field, to see which kind of expression it is + val typeConstraints = mutableListOf() + val callee = - this.handle(callExpr.`fun`) - ?: return ProblemExpression("Could not parse call expr without fun") + when (val `fun` = callExpr.`fun`) { + // If "fun" is either an index or an index list expression, this is a call with type + // constraints. We do not fully support that yet, but we can at least try to set + // some of the parameters as template parameters + is GoStandardLibrary.Ast.IndexExpr -> { + (frontend.typeOf(`fun`) as? ObjectType)?.generics?.let { typeConstraints += it } + this.handle(`fun`.x) + } + is GoStandardLibrary.Ast.IndexListExpr -> { + (frontend.typeOf(`fun`) as? ObjectType)?.generics?.let { typeConstraints += it } + this.handle(`fun`.x) + } + else -> { + this.handle(callExpr.`fun`) + } + } // Handle special functions, such as make and new in a special way val name = callee.name.localName @@ -172,20 +225,43 @@ class ExpressionHandler(frontend: GoLanguageFrontend) : } else { newCallExpression(callee, name, rawNode = callExpr) } + call.type = unknownType() + + // TODO(oxisto) Add type constraints + if (typeConstraints.isNotEmpty()) { + log.debug( + "Call {} has type constraints ({}), but we cannot add them to the call expression yet", + call.name, + typeConstraints.joinToString(", ") { it.name } + ) + } // Parse and add call arguments for (arg in callExpr.args) { - handle(arg)?.let { call += it } + call += handle(arg) } return call } + /** + * Unwrap is a small helper that unwraps an AST element that is wrapped into parenthesis. This + * is needed because we sometimes need to "peek" into the AST and this is not possible if the + * expression we are looking for is wrapped in parenthesis. + */ + private fun unwrap(expr: GoStandardLibrary.Ast.Expr): GoStandardLibrary.Ast.Expr { + return if (expr is GoStandardLibrary.Ast.ParenExpr) { + unwrap(expr.x) + } else { + expr + } + } + private fun handleKeyValueExpr( keyValueExpr: GoStandardLibrary.Ast.KeyValueExpr ): KeyValueExpression { - val key = handle(keyValueExpr.key) ?: newProblemExpression("could not parse key") - val value = handle(keyValueExpr.value) ?: newProblemExpression("could not parse value") + val key = handle(keyValueExpr.key) + val value = handle(keyValueExpr.value) return newKeyValueExpression(key, value, rawNode = keyValueExpr) } @@ -226,7 +302,7 @@ class ExpressionHandler(frontend: GoLanguageFrontend) : // second argument is a dimension (if this is an array), usually a literal if (args.size > 1) { - handle(args[1])?.let { array.addDimension(it) } + array.addDimension(handle(args[1])) } array } else { @@ -249,13 +325,16 @@ class ExpressionHandler(frontend: GoLanguageFrontend) : } private fun handleSelectorExpr(selectorExpr: GoStandardLibrary.Ast.SelectorExpr): Reference { - val base = handle(selectorExpr.x) ?: newProblemExpression("missing base") + val base = handle(selectorExpr.x) // Check, if this just a regular reference to a variable with a package scope and not a // member expression var isMemberExpression = true for (imp in frontend.currentFile?.imports ?: listOf()) { - if (base.name.localName == frontend.getImportName(imp)) { + // If we have an alias, we need to check it instead of the import name + val name = imp.name?.name ?: imp.importName + + if (base.name.localName == name) { // found a package name, so this is NOT a member expression isMemberExpression = false } @@ -282,9 +361,7 @@ class ExpressionHandler(frontend: GoLanguageFrontend) : */ private fun handleSliceExpr(sliceExpr: GoStandardLibrary.Ast.SliceExpr): SubscriptExpression { val ase = newSubscriptExpression(rawNode = sliceExpr) - ase.arrayExpression = - frontend.expressionHandler.handle(sliceExpr.x) - ?: newProblemExpression("missing array expression") + ase.arrayExpression = frontend.expressionHandler.handle(sliceExpr.x) // Build the slice expression val range = newRangeExpression(rawNode = sliceExpr) @@ -297,20 +374,39 @@ class ExpressionHandler(frontend: GoLanguageFrontend) : return ase } - private fun handleTypeAssertExpr( - typeAssertExpr: GoStandardLibrary.Ast.TypeAssertExpr - ): CastExpression { - val cast = newCastExpression(rawNode = typeAssertExpr) - - // Parse the inner expression - cast.expression = - handle(typeAssertExpr.x) ?: newProblemExpression("missing inner expression") + private fun handleStarExpr(starExpr: GoStandardLibrary.Ast.StarExpr): UnaryOperator { + val op = newUnaryOperator("*", postfix = false, prefix = false, rawNode = starExpr) + op.input = handle(starExpr.x) - // The type can be null, but only in certain circumstances, i.e, a type switch (which we do - // not support yet) - typeAssertExpr.type?.let { cast.castType = frontend.typeOf(it) } + return op + } - return cast + private fun handleTypeAssertExpr( + typeAssertExpr: GoStandardLibrary.Ast.TypeAssertExpr + ): Expression { + // This can either be a regular type assertion, which we handle as a cast expression or the + // "special" type assertion `.(type)`, which is used in a type switch to retrieve the type + // of the variable. In this case we treat it as a special unary operator. + return if (typeAssertExpr.type == null) { + val op = + newUnaryOperator( + ".(type)", + postfix = true, + prefix = false, + rawNode = typeAssertExpr + ) + op.input = handle(typeAssertExpr.x) + op + } else { + val cast = newCastExpression(rawNode = typeAssertExpr) + + // Parse the inner expression + cast.expression = handle(typeAssertExpr.x) + + // The type can be null, but only in certain circumstances, i.e, a type switch + typeAssertExpr.type?.let { cast.castType = frontend.typeOf(it) } + cast + } } private fun handleUnaryExpr(unaryExpr: GoStandardLibrary.Ast.UnaryExpr): UnaryOperator { @@ -321,7 +417,7 @@ class ExpressionHandler(frontend: GoLanguageFrontend) : prefix = false, rawNode = unaryExpr ) - handle(unaryExpr.x)?.let { op.input = it } + op.input = handle(unaryExpr.x) return op } @@ -331,31 +427,24 @@ class ExpressionHandler(frontend: GoLanguageFrontend) : * of a ConstructExpression and a list of KeyValueExpressions. The problem is that we need to * add the list as a first argument of the construct expression. */ - private fun handleCompositeLit( - compositeLit: GoStandardLibrary.Ast.CompositeLit - ): ConstructExpression { - // Parse the type field, to see which kind of expression it is - val type = frontend.typeOf(compositeLit.type) - - val construct = newConstructExpression(type.name, rawNode = compositeLit) - construct.type = type + private fun handleCompositeLit(compositeLit: GoStandardLibrary.Ast.CompositeLit): Expression { + // Parse the type field, to see which kind of expression it is. The type of a composite + // literal can be omitted if this is an "inner" composite literal, so we need to set it from + // the "outer" one. See below + val type = compositeLit.type?.let { frontend.typeOf(it) } ?: unknownType() val list = newInitializerListExpression(type, rawNode = compositeLit) - construct += list - - // Normally, the construct expression would not have DFG edge, but in this case we are - // mis-using it to simulate an object literal, so we need to add a DFG here, otherwise a - // declaration is disconnected from its initialization. - construct.addPrevDFG(list) + list.type = type val expressions = mutableListOf() for (elem in compositeLit.elts) { - handle(elem)?.let { expressions += it } + val expression = handle(elem) + expressions += expression } list.initializers = expressions - return construct + return list } /* diff --git a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoHandler.kt b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoHandler.kt index dc21160c7a..eeebac42a4 100644 --- a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoHandler.kt +++ b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoHandler.kt @@ -28,21 +28,26 @@ package de.fraunhofer.aisec.cpg.frontends.golang import de.fraunhofer.aisec.cpg.frontends.Handler import de.fraunhofer.aisec.cpg.graph.Node import de.fraunhofer.aisec.cpg.graph.ProblemNode +import de.fraunhofer.aisec.cpg.graph.statements.expressions.Literal import de.fraunhofer.aisec.cpg.helpers.Util import java.util.function.Supplier -abstract class GoHandler( - configConstructor: Supplier, +abstract class GoHandler( + configConstructor: Supplier, lang: GoLanguageFrontend -) : Handler(configConstructor, lang) { +) : Handler(configConstructor, lang) { /** * We intentionally override the logic of [Handler.handle] because we do not want the map-based * logic, but rather want to make use of the Kotlin-when syntax. * - * We also want non-nullable result handlers + * We also want to support nullable results for some handlers, e.g., when ignoring private + * declarations in dependencies. */ override fun handle(ctx: HandlerNode): ResultNode { val node = handleNode(ctx) + if (node == null) { + return node + } // The language frontend might set a location, which we should respect. Otherwise, we will // set the location here. @@ -77,6 +82,28 @@ abstract class GoHandler + val paths = (path?.value as? String)?.split("/") ?: listOf() + + // Return the last name in the path as the import name. However, if the last name is a + // module version (e.g., v5), then we need to return the second-to-last + var last = paths.lastOrNull() + last = + if (last?.startsWith("v") == true) { + paths.getOrNull(paths.size - 2) + } else { + last + } + + return last ?: "" + } } diff --git a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoLanguage.kt b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoLanguage.kt index eaa4dc6863..da8f93d80b 100644 --- a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoLanguage.kt +++ b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoLanguage.kt @@ -25,16 +25,19 @@ */ package de.fraunhofer.aisec.cpg.frontends.golang -import de.fraunhofer.aisec.cpg.frontends.HasGenerics -import de.fraunhofer.aisec.cpg.frontends.HasShortCircuitOperators -import de.fraunhofer.aisec.cpg.frontends.HasStructs -import de.fraunhofer.aisec.cpg.frontends.Language +import de.fraunhofer.aisec.cpg.frontends.* +import de.fraunhofer.aisec.cpg.graph.primitiveType import de.fraunhofer.aisec.cpg.graph.types.* import org.neo4j.ogm.annotation.Transient /** The Go language. */ class GoLanguage : - Language(), HasShortCircuitOperators, HasGenerics, HasStructs { + Language(), + HasShortCircuitOperators, + HasGenerics, + HasStructs, + HasFirstClassFunctions, + HasAnonymousIdentifier { override val fileExtensions = listOf("go") override val namespaceDelimiter = "." @Transient override val frontend = GoLanguageFrontend::class @@ -108,4 +111,37 @@ class GoLanguage : // https://pkg.go.dev/builtin#string "string" to StringType("string", this) ) + + override fun isDerivedFrom(type: Type, superType: Type): Boolean { + if (type == superType || superType == primitiveType("any")) { + return true + } + + // If we encounter an auto type as part of the function declaration, we accept this as any + // type + if ( + (type is ObjectType && superType is AutoType) || + (type is PointerType && type.isArray && superType.root is AutoType) + ) { + return true + } + + // We additionally want to emulate the behaviour of Go's interface system here + if (superType is ObjectType && superType.recordDeclaration?.kind == "interface") { + var b = true + val target = (type.root as? ObjectType)?.recordDeclaration + + // Our target struct type needs to implement all the functions of the interface + // TODO(oxisto): Differentiate on the receiver (pointer vs non-pointer) + for (method in superType.recordDeclaration?.methods ?: listOf()) { + if (target?.methods?.firstOrNull { it.signature == method.signature } != null) { + b = false + } + } + + return b + } + + return false + } } diff --git a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoLanguageFrontend.kt b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoLanguageFrontend.kt index 33fc7bb06c..33a06366d4 100644 --- a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoLanguageFrontend.kt +++ b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoLanguageFrontend.kt @@ -36,10 +36,11 @@ import de.fraunhofer.aisec.cpg.graph.* import de.fraunhofer.aisec.cpg.graph.declarations.DeclarationSequence import de.fraunhofer.aisec.cpg.graph.declarations.TranslationUnitDeclaration import de.fraunhofer.aisec.cpg.graph.newNamespaceDeclaration -import de.fraunhofer.aisec.cpg.graph.statements.expressions.Literal import de.fraunhofer.aisec.cpg.graph.types.FunctionType +import de.fraunhofer.aisec.cpg.graph.types.ObjectType import de.fraunhofer.aisec.cpg.graph.types.Type import de.fraunhofer.aisec.cpg.graph.unknownType +import de.fraunhofer.aisec.cpg.helpers.Util import de.fraunhofer.aisec.cpg.passes.EvaluationOrderGraphPass import de.fraunhofer.aisec.cpg.passes.GoEvaluationOrderGraphPass import de.fraunhofer.aisec.cpg.passes.GoExtraPass @@ -71,19 +72,82 @@ class GoLanguageFrontend(language: Language, ctx: Translatio private var commentMap: GoStandardLibrary.Ast.CommentMap? = null var currentFile: GoStandardLibrary.Ast.File? = null + var isDependency: Boolean = false + val declarationHandler = DeclarationHandler(this) val specificationHandler = SpecificationHandler(this) var statementHandler = StatementHandler(this) var expressionHandler = ExpressionHandler(this) + /** + * This helper class contains values needed to properly decide in which state const declaration + * / specifications are in. + */ + class DeclarationContext { + /** + * The current value of `iota`. This needs to be reset for each + * [GoStandardLibrary.Ast.GenDecl] and incremented for each observed + * [GoStandardLibrary.Ast.ValueSpec]. + */ + var iotaValue = -1 + + /** + * The current initializers in a list representing the different "columns". For example in + * the following code: + * ```go + * const ( + * a, b = 1, 2 + * c, d + * e, f = 4, 5 + * ) + * ``` + * + * The current list of initializers would first be (`1`,`2`) until a new set of initializers + * is declared in the last spec. The key corresponds to the "column" of the variable + * (a=0,b=1). + */ + var constInitializers = mutableMapOf() + + /** The current const type, which is valid until a new initializer is present */ + var constType: Type? = null + + /** The current [GoStandardLibrary.Ast.GenDecl] that is being processed. */ + var currentDecl: GoStandardLibrary.Ast.GenDecl? = null + } + + /** + * The current [DeclarationContext]. This is somewhat of a workaround since we cannot properly + * communicate state between different handlers. However, because *within* a [LanguageFrontend], + * everything is parsed sequentially according to AST order, we can safely use this context + * here. + */ + var declCtx = DeclarationContext() + @Throws(TranslationException::class) override fun parse(file: File): TranslationUnitDeclaration { + if (!shouldBeBuild(file, ctx.config.symbols)) { + log.debug( + "Ignoring the contents of {} because of missing build tags or different GOOS/GOARCH.", + file + ) + return newTranslationUnitDeclaration(file.name) + } + + val dependency = + ctx.config.includePaths.firstOrNull { + file.absolutePath.contains(it.toAbsolutePath().toString()) + } + // Make sure, that our top level is set either way val topLevel = - if (config.topLevel != null) { - config.topLevel - } else { - file.parentFile + // If this file is part of an include, we set the top level to the root of the include + when { + dependency != null -> { + isDependency = true + dependency.toFile() + } + config.topLevel != null -> config.topLevel + else -> file.parentFile }!! val std = GoStandardLibrary.INSTANCE @@ -95,7 +159,7 @@ class GoLanguageFrontend(language: Language, ctx: Translatio } val fset = std.NewFileSet() - val f = Parser.parseFile(fset, file.absolutePath, file.readText()) + val f = Parser.parseFile(fset, file.absolutePath) this.commentMap = std.NewCommentMap(fset, f, f.comments) @@ -119,8 +183,12 @@ class GoLanguageFrontend(language: Language, ctx: Translatio // module path as well as the current directory in relation to the topLevel var packagePath = file.parentFile.relativeTo(topLevel) - // If we are in a module, we need to prepend the module path to it - currentModule?.let { packagePath = File(it.module.mod.path).resolve(packagePath) } + // If we are in a module, we need to prepend the module path to it. There is an + // exception if we are in the "std" module, which represents the standard library + val modulePath = currentModule?.module?.mod?.path + if (modulePath != null && modulePath != "std") { + packagePath = File(modulePath).resolve(packagePath) + } p.path = packagePath.path } catch (ex: IllegalArgumentException) { @@ -149,72 +217,161 @@ class GoLanguageFrontend(language: Language, ctx: Translatio } override fun typeOf(type: GoStandardLibrary.Ast.Expr): Type { - return when (type) { - is GoStandardLibrary.Ast.Ident -> { - val name: String = - if (isBuiltinType(type.name)) { - // Definitely not an FQN type - type.name - } else { - // FQN'ize this name (with the current file) - "${currentFile?.name?.name}.${type.name}" // this.File.Name.Name + val type = + when (type) { + is GoStandardLibrary.Ast.Ident -> { + val name: String = + if (isBuiltinType(type.name)) { + // Definitely not an FQN type + type.name + } else { + // FQN'ize this name (with the current file) + "${currentFile?.name?.name}.${type.name}" // this.File.Name.Name + } + + objectType(name) + } + is GoStandardLibrary.Ast.SelectorExpr -> { + // This is a FQN type + val baseName = (type.x as? GoStandardLibrary.Ast.Ident)?.name?.let { Name(it) } + + return objectType(Name(type.sel.name, baseName)) + } + is GoStandardLibrary.Ast.ArrayType -> { + return typeOf(type.elt).array() + } + is GoStandardLibrary.Ast.ChanType -> { + // Handle them similar to a map type (see below) + return objectType("chan", listOf(typeOf(type.value))) + } + is GoStandardLibrary.Ast.FuncType -> { + val paramTypes = + type.params.list + .flatMap { field -> + // Because we can have unnamed parameters or multiple parameters + // declared at once, we need to expand the list of types according + // to the list of names + if (field.names.isEmpty()) { + listOf(field.type) + } else { + field.names.map { field.type } + } + } + .map { fieldTypeOf(it).first } + val returnTypes = type.results?.list?.map { typeOf(it.type) } ?: listOf() + val name = funcTypeName(paramTypes, returnTypes) + + FunctionType(name, paramTypes, returnTypes, this.language) + } + is GoStandardLibrary.Ast.IndexExpr -> { + // A go type constraint, aka generic + val baseType = typeOf(type.x) + val generics = listOf(typeOf(type.index)) + objectType(baseType.name, generics) + } + is GoStandardLibrary.Ast.IndexListExpr -> { + // A go type constraint, aka generic with multiple types + val baseType = typeOf(type.x) + val generics = type.indices.map { typeOf(it) } + objectType(baseType.name, generics) + } + is GoStandardLibrary.Ast.StructType -> { + // Go allows to use anonymous structs as type. This is something we cannot model + // properly in the CPG yet. In order to at least deal with this partially, we + // construct a ObjectType and put the fields and their types into the type. + // This will result in something like `struct{name string; args util.args; want + // string}` + val parts = + type.fields.list.map { field -> + var desc = "" + // Name can be optional, if its embedded + field.names.getOrNull(0)?.let { desc += it } + desc += " " + desc += fieldTypeOf(field.type).first.name + desc + } + + val name = parts.joinToString("; ", "struct{", "}") + + // Create an anonymous struct, this will add it to the scope manager. This is + // somewhat duplicate, but the easiest for now. We need to create it in the + // global + // scope to avoid namespace issues + var record = + scopeManager.withScope(scopeManager.globalScope) { + specificationHandler.buildRecordDeclaration(type, name) + } + + record.toType() + } + is GoStandardLibrary.Ast.InterfaceType -> { + // Go allows to use anonymous interface as type. This is something we cannot + // model + // properly in the CPG yet. In order to at least deal with this partially, we + // construct a ObjectType and put the methods and their types into the type. + + // In the easiest case this is the empty interface `interface{}`, which we then + // consider to be the "any" type. `any` is actually a type alias for + // `interface{}`, + // but in modern Go `any` is preferred. + if (type.methods.list.isEmpty()) { + return primitiveType("any") } - objectType(name) + val parts = + type.methods.list.map { method -> + var desc = "" + // Name can be optional, if its embedded + method.names.getOrNull(0)?.let { desc += it } + // the function type has a weird "func" prefix, which we do not want + desc += typeOf(method.type).name.toString().removePrefix("func") + desc + } + + objectType(parts.joinToString("; ", "interface{", "}")) + } + is GoStandardLibrary.Ast.MapType -> { + // We cannot properly represent Go's built-in map types, yet so we have + // to make a shortcut here and represent it as a Java-like map type. + return objectType("map", listOf(typeOf(type.key), typeOf(type.value))) + } + is GoStandardLibrary.Ast.StarExpr -> { + typeOf(type.x).pointer() + } + else -> { + Util.warnWithFileLocation( + this, + type, + log, + "Not parsing type of type ${type.goType} yet" + ) + unknownType() + } } - is GoStandardLibrary.Ast.ArrayType -> { - return typeOf(type.elt).array() - } - is GoStandardLibrary.Ast.ChanType -> { - // Handle them similar to a map type (see below) - return objectType("chan", listOf(typeOf(type.value))) - } - is GoStandardLibrary.Ast.FuncType -> { - val paramTypes = type.params.list.map { typeOf(it.type) } - val returnTypes = type.results?.list?.map { typeOf(it.type) } ?: listOf() - val name = funcTypeName(paramTypes, returnTypes) - return FunctionType(name, paramTypes, returnTypes, this.language) - } - is GoStandardLibrary.Ast.MapType -> { - // We cannot properly represent Go's built-in map types, yet so we have - // to make a shortcut here and represent it as a Java-like map type. - return objectType("map", listOf(typeOf(type.key), typeOf(type.value))) - } - is GoStandardLibrary.Ast.StarExpr -> { - typeOf(type.x).pointer() - } - else -> { - log.warn("Not parsing type of type ${type.goType} yet") - unknownType() + return typeManager.registerType(typeManager.resolvePossibleTypedef(type, scopeManager)) + } + + /** + * A quick helper function to retrieve the type of a field, to check for possible variadic + * arguments. + */ + internal fun fieldTypeOf( + paramType: GoStandardLibrary.Ast.Expr, + ): Pair { + var variadic = false + val type = + if (paramType is GoStandardLibrary.Ast.Ellipsis) { + variadic = true + typeOf(paramType.elt).array() + } else { + typeOf(paramType) } - } + return Pair(type, variadic) } private fun isBuiltinType(name: String): Boolean { - return when (name) { - "bool", - "byte", - "complex128", - "complex64", - "error", - "float32", - "float64", - "int", - "int8", - "int16", - "int32", - "int64", - "rune", - "string", - "uint", - "uint8", - "uint16", - "uint32", - "uint64", - "uintptr" -> true - else -> false - } + return language.primitiveTypeNames.contains(name) } override fun codeOf(astNode: GoStandardLibrary.Ast.Node): String? { @@ -239,43 +396,90 @@ class GoLanguageFrontend(language: Language, ctx: Translatio } } - /** - * This function produces a Go-style function type name such as `func(int, string) string` or - * `func(int) (error, string)` - */ - private fun funcTypeName(paramTypes: List, returnTypes: List): String { - val rn = mutableListOf() - val pn = mutableListOf() - - for (t in paramTypes) { - pn += t.name.toString() - } + companion object { + /** + * All possible goos values. See + * https://github.com/golang/go/blob/release-branch.go1.21/src/go/build/syslist.go#L11 + */ + val goosValues = + listOf( + "aix", + "android", + "darwin", + "dragonfly", + "freebsd", + "hurd", + "illumos", + "ios", + "js", + "linux", + "nacl", + "netbsd", + "openbsd", + "plan9", + "solaris", + "wasip1", + "windows", + "zos" + ) - for (t in returnTypes) { - rn += t.name.toString() - } + /** + * All possible architecture values. See + * https://github.com/golang/go/blob/release-branch.go1.21/src/go/build/syslist.go#L54 + */ + val goarchValues = + listOf( + "386", + "amd64", + "arm", + "arm64", + "loong64", + "mips", + "mips64", + "mips64le", + "mipsle", + "ppc64", + "ppc64le", + "riscv64", + "s390x" + ) + } +} - val rs = - if (returnTypes.size > 1) { - rn.joinToString(", ", prefix = " (", postfix = ")") - } else if (returnTypes.isNotEmpty()) { - rn.joinToString(", ", prefix = " ") - } else { - "" - } +val Type?.underlyingType: Type? + get() { + return (this as? ObjectType)?.recordDeclaration?.superClasses?.singleOrNull() + } - return pn.joinToString(", ", prefix = "func(", postfix = ")$rs") +val Type?.isOverlay: Boolean + get() { + return this is ObjectType && this.recordDeclaration?.kind == "overlay" } - fun getImportName(spec: GoStandardLibrary.Ast.ImportSpec): String { - val name = spec.name - if (name != null) { - return name.name - } +/** + * This function produces a Go-style function type name such as `func(int, string) string` or + * `func(int) (error, string)` + */ +fun funcTypeName(paramTypes: List, returnTypes: List): String { + val rn = mutableListOf() + val pn = mutableListOf() - val path = expressionHandler.handle(spec.path) as? Literal<*> - val paths = (path?.value as? String)?.split("/") ?: listOf() + for (t in paramTypes) { + pn += t.name.toString() + } - return paths.lastOrNull() ?: "" + for (t in returnTypes) { + rn += t.name.toString() } + + val rs = + if (returnTypes.size > 1) { + rn.joinToString(", ", prefix = " (", postfix = ")") + } else if (returnTypes.isNotEmpty()) { + rn.joinToString(", ", prefix = " ") + } else { + "" + } + + return pn.joinToString(", ", prefix = "func(", postfix = ")$rs") } diff --git a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoStandardLibrary.kt b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoStandardLibrary.kt index f7ca20b70b..0fc6480d57 100644 --- a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoStandardLibrary.kt +++ b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoStandardLibrary.kt @@ -52,8 +52,8 @@ interface GoStandardLibrary : Library { * this package. */ object Parser { - fun parseFile(fileSet: Ast.FileSet, path: String, contents: String): Ast.File { - return INSTANCE.goParserParseFile(fileSet, path, contents) + fun parseFile(fileSet: Ast.FileSet, path: String): Ast.File { + return INSTANCE.goParserParseFile(fileSet, path) } } @@ -153,6 +153,11 @@ interface GoStandardLibrary : Library { get() { return list(INSTANCE::GetNumGenDeclSpecs, INSTANCE::GetGenDeclSpec) } + + val tok: Int + get() { + return INSTANCE.GetGenDeclTok(this) + } } class FuncDecl(p: Pointer? = Pointer.NULL) : Decl(p) { @@ -168,7 +173,7 @@ interface GoStandardLibrary : Library { return INSTANCE.GetFuncDeclName(this) } - val body: BlockStmt + val body: BlockStmt? get() { return INSTANCE.GetFuncDeclBody(this) } @@ -195,6 +200,11 @@ interface GoStandardLibrary : Library { return INSTANCE.GetTypeSpecName(this) } + val assign: Int + get() { + return INSTANCE.GetTypeSpecAssign(this) + } + val type: Expr get() { return INSTANCE.GetTypeSpecType(this) @@ -243,7 +253,9 @@ interface GoStandardLibrary : Library { "*ast.FuncLit" -> FuncLit(nativeValue) "*ast.Ident" -> Ident(nativeValue) "*ast.IndexExpr" -> IndexExpr(nativeValue) + "*ast.IndexListExpr" -> IndexListExpr(nativeValue) "*ast.KeyValueExpr" -> KeyValueExpr(nativeValue) + "*ast.ParenExpr" -> ParenExpr(nativeValue) "*ast.SelectorExpr" -> SelectorExpr(nativeValue) "*ast.StarExpr" -> StarExpr(nativeValue) "*ast.SliceExpr" -> SliceExpr(nativeValue) @@ -311,7 +323,7 @@ interface GoStandardLibrary : Library { } class CompositeLit(p: Pointer? = Pointer.NULL) : Expr(p) { - val type: Expr + val type: Expr? get() { return INSTANCE.GetCompositeLitType(this) } @@ -334,6 +346,13 @@ interface GoStandardLibrary : Library { } } + class ParenExpr(p: Pointer? = Pointer.NULL) : Expr(p) { + val x: Expr + get() { + return INSTANCE.GetParenExprX(this) + } + } + class FuncLit(p: Pointer? = Pointer.NULL) : Expr(p) { fun toDecl(): FuncDecl { return INSTANCE.MakeFuncDeclFromFuncLit(this) @@ -345,6 +364,11 @@ interface GoStandardLibrary : Library { } class Ident(p: Pointer? = Pointer.NULL) : Expr(p) { + val isUnexported: Boolean + get() { + return name.isNotEmpty() && name[0].isLowerCase() + } + val name: String get() { return INSTANCE.GetIdentName(this) @@ -367,6 +391,21 @@ interface GoStandardLibrary : Library { } } + class IndexListExpr(p: Pointer? = Pointer.NULL) : Expr(p) { + val x: Expr + get() { + return INSTANCE.GetIndexListExprX(this) + } + + val indices: List + get() { + return list( + INSTANCE::GetNumIndexListExprIndices, + INSTANCE::GetIndexListExprIndex + ) + } + } + class SelectorExpr(p: Pointer? = Pointer.NULL) : Expr(p) { val x: Expr get() { @@ -517,7 +556,9 @@ interface GoStandardLibrary : Library { "*ast.LabeledStmt" -> LabeledStmt(nativeValue) "*ast.RangeStmt" -> RangeStmt(nativeValue) "*ast.ReturnStmt" -> ReturnStmt(nativeValue) + "*ast.SendStmt" -> SendStmt(nativeValue) "*ast.SwitchStmt" -> SwitchStmt(nativeValue) + "*ast.TypeSwitchStmt" -> TypeSwitchStmt(nativeValue) else -> super.fromNative(nativeValue, context) } } @@ -700,6 +741,18 @@ interface GoStandardLibrary : Library { } } + class SendStmt(p: Pointer? = Pointer.NULL) : Stmt(p) { + val chan: Expr + get() { + return INSTANCE.GetSendStmtChan(this) + } + + val value: Expr + get() { + return INSTANCE.GetSendStmtValue(this) + } + } + class SwitchStmt(p: Pointer? = Pointer.NULL) : Stmt(p) { val init: Stmt? get() { @@ -717,6 +770,23 @@ interface GoStandardLibrary : Library { } } + class TypeSwitchStmt(p: Pointer? = Pointer.NULL) : Stmt(p) { + val init: Stmt? + get() { + return INSTANCE.GetTypeSwitchStmtInit(this) + } + + val assign: Stmt + get() { + return INSTANCE.GetTypeSwitchStmtAssign(this) + } + + val body: BlockStmt + get() { + return INSTANCE.GetTypeSwitchStmtBody(this) + } + } + class Position(p: Pointer? = Pointer.NULL) : GoObject(p) { val line: Int get() { @@ -774,7 +844,7 @@ interface GoStandardLibrary : Library { // go/parser package - fun goParserParseFile(fileSet: Ast.FileSet, path: String, src: String): Ast.File + fun goParserParseFile(fileSet: Ast.FileSet, path: String): Ast.File fun GetType(obj: Pointer): String @@ -826,9 +896,9 @@ interface GoStandardLibrary : Library { fun GetFuncDeclName(funcDecl: Ast.FuncDecl): Ast.Ident - fun GetFuncDeclBody(funcDecl: Ast.FuncDecl): Ast.BlockStmt + fun GetFuncDeclBody(funcDecl: Ast.FuncDecl): Ast.BlockStmt? - fun GetCompositeLitType(compositeLit: Ast.CompositeLit): Ast.Expr + fun GetCompositeLitType(compositeLit: Ast.CompositeLit): Ast.Expr? fun GetNumCompositeLitElts(compositeLit: Ast.CompositeLit): Int @@ -844,6 +914,8 @@ interface GoStandardLibrary : Library { fun GetKeyValueExprValue(keyValueExpr: Ast.KeyValueExpr): Ast.Expr + fun GetParenExprX(parenExpr: Ast.ParenExpr): Ast.Expr + fun GetBasicLitValue(basicLit: Ast.BasicLit): String fun GetBasicLitKind(basicLit: Ast.BasicLit): Int @@ -958,6 +1030,12 @@ interface GoStandardLibrary : Library { fun GetIndexExprIndex(IndexExpr: Ast.IndexExpr): Ast.Expr + fun GetIndexListExprX(indexListExpr: Ast.IndexListExpr): Ast.Expr + + fun GetNumIndexListExprIndices(indexListExpr: Ast.IndexListExpr): Int + + fun GetIndexListExprIndex(indexListExpr: Ast.IndexListExpr, i: Int): Ast.Expr + fun GetIfStmtInit(ifStmt: Ast.IfStmt): Ast.Stmt? fun GetIfStmtCond(ifStmt: Ast.IfStmt): Ast.Expr @@ -980,16 +1058,28 @@ interface GoStandardLibrary : Library { fun GetReturnStmtResult(returnStmt: Ast.ReturnStmt, i: Int): Ast.Expr + fun GetSendStmtChan(sendStmt: Ast.SendStmt): Ast.Expr + + fun GetSendStmtValue(sendStmt: Ast.SendStmt): Ast.Expr + fun GetSwitchStmtInit(switchStmt: Ast.SwitchStmt): Ast.Stmt? fun GetSwitchStmtTag(switchStmt: Ast.SwitchStmt): Ast.Expr? fun GetSwitchStmtBody(stmt: Ast.SwitchStmt): Ast.BlockStmt + fun GetTypeSwitchStmtInit(typeSwitchStmt: Ast.TypeSwitchStmt): Ast.Stmt? + + fun GetTypeSwitchStmtAssign(typeSwitchStmt: Ast.TypeSwitchStmt): Ast.Stmt + + fun GetTypeSwitchStmtBody(typeSwitchStmt: Ast.TypeSwitchStmt): Ast.BlockStmt + fun GetNumGenDeclSpecs(genDecl: Ast.GenDecl): Int fun GetGenDeclSpec(genDecl: Ast.GenDecl, i: Int): Ast.Spec + fun GetGenDeclTok(genDecl: Ast.GenDecl): Int + fun GetImportSpecName(importSpec: Ast.ImportSpec): Ast.Ident? fun GetImportSpecPath(importSpec: Ast.ImportSpec): Ast.BasicLit @@ -998,7 +1088,7 @@ interface GoStandardLibrary : Library { fun GetValueSpecName(valueSpec: Ast.ValueSpec, i: Int): Ast.Ident - fun GetValueSpecType(valueSpec: Ast.ValueSpec): Ast.Expr + fun GetValueSpecType(valueSpec: Ast.ValueSpec): Ast.Expr? fun GetNumValueSpecValues(valueSpec: Ast.ValueSpec): Int @@ -1006,6 +1096,8 @@ interface GoStandardLibrary : Library { fun GetTypeSpecName(typeSpec: Ast.TypeSpec): Ast.Ident + fun GetTypeSpecAssign(typeSpec: Ast.TypeSpec): Int + fun GetTypeSpecType(typeSpec: Ast.TypeSpec): Ast.Expr companion object { diff --git a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoUtils.kt b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoUtils.kt new file mode 100644 index 0000000000..a06a2471b5 --- /dev/null +++ b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoUtils.kt @@ -0,0 +1,234 @@ +/* + * Copyright (c) 2023, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.frontends.golang + +import de.fraunhofer.aisec.cpg.TranslationManager +import de.fraunhofer.aisec.cpg.TranslationResult +import de.fraunhofer.aisec.cpg.frontends.CompilationDatabase +import java.io.File +import java.util.concurrent.TimeUnit +import org.slf4j.Logger +import org.slf4j.LoggerFactory + +/** + * This functions checks whether the file specified in [file] should be processed in the + * [GoLanguageFrontend]. It mainly checks for build constraints, which can either be part of the + * filename or specified in the file (see [BuildConstraintExpression]). + * + * Note: This is currently specific to Go, but could be integrated into the [TranslationManager] for + * other languages using an appropriate interface. + */ +internal fun shouldBeBuild(file: File, symbols: Map): Boolean { + // First, we need to check, whether the filename ends with a possible _$GOOS.go, $_GOARCH.go + // or $_GOOS_$GOARCH.go + val parts = file.nameWithoutExtension.split("_") + + // If the last part is a possible GOOS value, then we need to check, if this is equal to our + // current GOOS + if ( + parts.lastOrNull() in GoLanguageFrontend.goosValues && parts.lastOrNull() != symbols["GOOS"] + ) { + // Skip the contents + return false + } + + // If the last part is a possible GOARCH value, then we need to check, if this is equal to + // our current GOARCH + if ( + parts.lastOrNull() in GoLanguageFrontend.goarchValues && + parts.lastOrNull() != symbols["GOARCH"] + ) { + return false + } + + val sub = parts.subList((parts.size - 2).coerceAtLeast(0), parts.size.coerceAtLeast(0)) + if ( + sub.size == 2 && + ((sub[0] in GoLanguageFrontend.goosValues && sub[0] != symbols["GOOS"]) || + (sub[1] in GoLanguageFrontend.goarchValues && sub[1] != symbols["GOARCH"])) + ) { + return false + } + + // Next, we need to peek into the file, to see whether any build tags are present. The + // fastest way + // to do that is to read the file and look for a go:build line + val goBuildLine = + file + .bufferedReader() + .useLines { lines -> lines.take(50).toList() } + .firstOrNull { it.startsWith("//go:build") } + ?: return true + + val constraint = BuildConstraintExpression.fromString(goBuildLine.substringAfter("//go:build ")) + + return constraint?.evaluate(symbols.buildTags) == true +} + +private val Map.buildTags: Set + get() { + val tags = mutableSetOf() + val goos = this["GOOS"] + val goarch = this["GOARCH"] + + // Add GOOS and GOARCH + goos?.let { tags += it } + goarch?.let { tags += it } + + // We need to derive some more build tags based on the GOOS. + // See + // https://github.com/golang/go/blob/release-branch.go1.21/src/go/build/syslist.go#L39 + if ( + goos in + listOf( + "aix", + "android", + "darwin", + "dragonfly", + "freebsd", + "hurd", + "illumos", + "ios", + "linux", + "netbsd", + "openbsd", + "solaris" + ) + ) { + tags += "unix" + } + + // Additional "derived" operating systems + when (goos) { + "android" -> tags += "linux" + "illumos" -> tags += "solaris" + "ios" -> tags += "darwin" + } + + // Add remaining tags + this["-tags"]?.split(" ")?.let { tags += it } + + return tags + } + +internal fun gatherGoFiles(root: File, includeSubDir: Boolean = true): List { + return root + .walkTopDown() + .onEnter { (it == root || includeSubDir) && !it.name.contains(".go") } + .filter { + // skip tests for now + it.extension == "go" && !it.name.endsWith("_test.go") + } + .toList() +} + +/** + * Represents a Go project. This could potentially be extended and moved to cpg-core to be available + * for other languages. It shares some fields with a [CompilationDatabase] and both could + * potentially be merged. + */ +internal class Project { + var symbols: Map = mutableMapOf() + + var components: MutableMap> = mutableMapOf() + + var includePaths: List = mutableListOf() + + companion object { + val log: Logger = LoggerFactory.getLogger(Project::class.java) + + /** + * This function emulates building a Go project. It requires an installed Go environment and + * uses the Go binary to compile a list of package dependencies, which are then included + * into the [Project] as includes. + * + * Note: This currently is limited to packages of the standard library + */ + internal fun buildProject( + modulePath: String, + goos: String? = null, + goarch: String? = null, + tags: List = listOf() + ): Project { + val project = Project() + val symbols = mutableMapOf() + var files = mutableListOf() + + val topLevel = File(modulePath) + + val pb = + ProcessBuilder("go", "list", tags.joinToString(",", "-tags="), "all") + .directory(topLevel) + .redirectOutput(ProcessBuilder.Redirect.PIPE) + + val env = pb.environment() + env["GOOS"] = goos + env["GOARCH"] = goarch + + var proc = pb.start() + proc.waitFor(5, TimeUnit.MINUTES) + if (proc.exitValue() != 0) { + log.debug(proc.errorStream.bufferedReader().readLine()) + } + + // For now, we only support deps in the standard library + val deps = proc.inputStream.bufferedReader().readLines().filter { !it.contains(".") } + + log.debug("Identified {} package dependencies (stdlib only)", deps.size) + + proc = + ProcessBuilder("go", "env", "GOROOT") + .redirectOutput(ProcessBuilder.Redirect.PIPE) + .start() + proc.waitFor(5, TimeUnit.MINUTES) + if (proc.exitValue() != 0) { + log.debug(proc.errorStream.bufferedReader().readLine()) + } + + val stdLib = File(proc.inputStream.bufferedReader().readLine()).resolve("src") + + log.debug("GOROOT/src is located @ {}", stdLib) + + files += deps.flatMap { gatherGoFiles(stdLib.resolve(it), false) } + files += gatherGoFiles(topLevel) + + goos?.let { symbols["GOOS"] = it } + goarch?.let { symbols["GOARCH"] = it } + tags.let { symbols["-tags"] = tags.joinToString { " " } } + + // Pre-filter any files we are not building anyway based on our symbols + files = files.filter { shouldBeBuild(it, symbols) }.toMutableList() + + // TODO(oxisto): look for binaries in cmd folder + project.components[TranslationResult.APPLICATION_LOCAL_NAME] = files + project.symbols = symbols + // TODO(oxisto): support vendor includes + project.includePaths = listOf(stdLib) + + return project + } + } +} diff --git a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/SpecificationHandler.kt b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/SpecificationHandler.kt index 8dd43ab566..cf17fde0ac 100644 --- a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/SpecificationHandler.kt +++ b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/SpecificationHandler.kt @@ -25,40 +25,29 @@ */ package de.fraunhofer.aisec.cpg.frontends.golang -import de.fraunhofer.aisec.cpg.frontends.Handler -import de.fraunhofer.aisec.cpg.frontends.HandlerInterface import de.fraunhofer.aisec.cpg.graph.* import de.fraunhofer.aisec.cpg.graph.declarations.* +import de.fraunhofer.aisec.cpg.graph.scopes.NameScope +import de.fraunhofer.aisec.cpg.helpers.Util class SpecificationHandler(frontend: GoLanguageFrontend) : - Handler( - ::ProblemDeclaration, - frontend - ) { - - init { - map[GoStandardLibrary.Ast.ImportSpec::class.java] = HandlerInterface { - handleImportSpec(it as GoStandardLibrary.Ast.ImportSpec) - } - map[GoStandardLibrary.Ast.TypeSpec::class.java] = HandlerInterface { - handleTypeSpec(it as GoStandardLibrary.Ast.TypeSpec) - } - map[GoStandardLibrary.Ast.ValueSpec::class.java] = HandlerInterface { - handleValueSpec(it as GoStandardLibrary.Ast.ValueSpec) - } - map.put(GoStandardLibrary.Ast.Spec::class.java, ::handleNode) - } + GoHandler(::ProblemDeclaration, frontend) { - private fun handleNode(spec: GoStandardLibrary.Ast.Spec): Declaration { - val message = "Not parsing specification of type ${spec.goType} yet" - log.error(message) - - return newProblemDeclaration(message) + override fun handleNode(node: GoStandardLibrary.Ast.Spec): Declaration? { + return when (node) { + is GoStandardLibrary.Ast.ImportSpec -> handleImportSpec(node) + is GoStandardLibrary.Ast.TypeSpec -> handleTypeSpec(node) + is GoStandardLibrary.Ast.ValueSpec -> handleValueSpec(node) + else -> { + return handleNotSupported(node, node.goType) + } + } } private fun handleImportSpec(importSpec: GoStandardLibrary.Ast.ImportSpec): IncludeDeclaration { // We set the name of the include declaration to the imported name, i.e., the package name - val name = frontend.getImportName(importSpec) + val name = importSpec.importName + // We set the filename of the include declaration to the package path, i.e., its full path // including any module identifiers. This way we can match the include declaration back to // the namespace's path and name @@ -66,6 +55,17 @@ class SpecificationHandler(frontend: GoLanguageFrontend) : val include = newIncludeDeclaration(filename, rawNode = importSpec) include.name = parseName(name) + var alias = importSpec.name?.name + if (alias != null && alias != name) { + var location = frontend.locationOf(importSpec) + + // If the name differs from the import name, we have an alias + // Add an alias for the package on the current file + location?.artifactLocation?.let { + frontend.scopeManager.addAlias(it, Name(name), Name(alias)) + } + } + return include } @@ -75,7 +75,14 @@ class SpecificationHandler(frontend: GoLanguageFrontend) : when (type) { is GoStandardLibrary.Ast.StructType -> handleStructTypeSpec(spec, type) is GoStandardLibrary.Ast.InterfaceType -> handleInterfaceTypeSpec(spec, type) - else -> return ProblemDeclaration("not parsing type of type ${spec.goType} yet") + is GoStandardLibrary.Ast.FuncType -> handleFuncTypeSpec(spec, type) + is GoStandardLibrary.Ast.Ident, + is GoStandardLibrary.Ast.SelectorExpr, + is GoStandardLibrary.Ast.MapType, + is GoStandardLibrary.Ast.ArrayType, + is GoStandardLibrary.Ast.StarExpr, + is GoStandardLibrary.Ast.ChanType -> handleTypeDef(spec, type) + else -> return ProblemDeclaration("not parsing type of type ${type.goType} yet") } return decl @@ -85,26 +92,39 @@ class SpecificationHandler(frontend: GoLanguageFrontend) : typeSpec: GoStandardLibrary.Ast.TypeSpec, structType: GoStandardLibrary.Ast.StructType ): RecordDeclaration { - val record = newRecordDeclaration(typeSpec.name.name, "struct", rawNode = typeSpec) + val record = buildRecordDeclaration(structType, typeSpec.name.name, typeSpec) + + // Make sure to register the type + frontend.typeManager.registerType(record.toType()) + + return record + } + + fun buildRecordDeclaration( + structType: GoStandardLibrary.Ast.StructType, + name: CharSequence, + typeSpec: GoStandardLibrary.Ast.TypeSpec? = null, + ): RecordDeclaration { + val record = newRecordDeclaration(name, "struct", rawNode = typeSpec) frontend.scopeManager.enterScope(record) if (!structType.incomplete) { for (field in structType.fields.list) { - // a field can also have no name, which means that it is embedded, not quite - // sure yet how to handle this, but since the embedded field can be accessed - // by its type, it could make sense to name the field according to the type val type = frontend.typeOf(field.type) - val name = + // A field can also have no name, which means that it is embedded. In this case, it + // can be accessed by the local name of its type and therefore we name the field + // accordingly + val fieldName = if (field.names.isEmpty()) { - // Retrieve the root type name - type.root.name.toString() + // Retrieve the root type local name + type.root.name.localName } else { field.names[0].name } - val decl = newFieldDeclaration(name, type, rawNode = field) + val decl = newFieldDeclaration(fieldName, type, rawNode = field) frontend.scopeManager.addDeclaration(decl) } } @@ -120,6 +140,9 @@ class SpecificationHandler(frontend: GoLanguageFrontend) : ): Declaration { val record = newRecordDeclaration(typeSpec.name.name, "interface", rawNode = typeSpec) + // Make sure to register the type + frontend.typeManager.registerType(record.toType()) + frontend.scopeManager.enterScope(record) if (!interfaceType.incomplete) { @@ -151,39 +174,165 @@ class SpecificationHandler(frontend: GoLanguageFrontend) : } /** - * // handleValueSpec handles parsing of an ast.ValueSpec, which is a variable // declaration. - * Since this can potentially declare multiple variables with one // "spec", this returns a + * // handleValueSpec handles parsing of an ast.ValueSpec, which is a variable declaration. + * Since this can potentially declare multiple variables with one "spec", this returns a * [DeclarationSequence]. */ - private fun handleValueSpec(valueSpec: GoStandardLibrary.Ast.ValueSpec): DeclarationSequence { - val sequence = DeclarationSequence() + private fun handleValueSpec( + valueSpec: GoStandardLibrary.Ast.ValueSpec, + ): Declaration { + // Increment iota value + frontend.declCtx.iotaValue++ - for ((idx, ident) in valueSpec.names.withIndex()) { - val decl = newVariableDeclaration(ident.name, rawNode = valueSpec) + // If we only have one initializer on the right side and multiple ones on the left side, + // we are deconstructing a tuple + val lenValues = valueSpec.values.size + if (lenValues == 1 && lenValues != valueSpec.names.size) { + // We need to construct a "tuple" declaration on the left side that holds all the + // variables + val tuple = TupleDeclaration() + tuple.type = autoType() - if (valueSpec.type != null) { - decl.type = frontend.typeOf(valueSpec.type!!) - } else { - decl.type = autoType() - } + for (ident in valueSpec.names) { + // We want to make sure that top-level declarations, i.e, the ones that are directly + // in a namespace are FQNs. Otherwise we cannot resolve them properly when we access + // them outside of the package. + val fqn = + if (frontend.scopeManager.currentScope is NameScope) { + fqn(ident.name) + } else { + ident.name + } + val decl = newVariableDeclaration(fqn, rawNode = valueSpec) + + if (valueSpec.type != null) { + decl.type = frontend.typeOf(valueSpec.type!!) + } else { + decl.type = autoType() + } + + if (valueSpec.values.isNotEmpty()) { + tuple.initializer = frontend.expressionHandler.handle(valueSpec.values[0]) + } + + // We need to manually add the variables to the scope manager + frontend.scopeManager.addDeclaration(decl) - // There could either be no initializers, otherwise the amount of values - // must match the names - val lenValues = valueSpec.values.size - if (lenValues != 0 && lenValues != valueSpec.names.size) { - log.error( - "Number of initializers does not match number of names. Initializers might be incomplete" - ) + tuple += decl } + return tuple + } else { + val sequence = DeclarationSequence() + + var type = valueSpec.type?.let { frontend.typeOf(it) } + + for ((nameIdx, ident) in valueSpec.names.withIndex()) { + // We want to make sure that top-level declarations, i.e, the ones that are directly + // in a namespace are FQNs. Otherwise we cannot resolve them properly when we access + // them outside of the package. + val fqn = + if (frontend.scopeManager.currentScope is NameScope) { + fqn(ident.name) + } else { + ident.name + } + val decl = newVariableDeclaration(fqn, rawNode = valueSpec) + if (type != null) { + decl.type = type + } else { + decl.type = autoType() + } - // The initializer is in the "Values" slice with the respective index - if (valueSpec.values.size > idx) { - decl.initializer = frontend.expressionHandler.handle(valueSpec.values[idx]) + if (valueSpec.values.size > nameIdx) { + // the initializer is in the "Values" slice with the respective index + decl.initializer = frontend.expressionHandler.handle(valueSpec.values[nameIdx]) + } + + // If we are in a const declaration, we need to do something rather unusual. + // If we have an initializer, we need to set this as the current const initializer, + // because following specs will "inherit" the one from the previous line. + // + // Note: we cannot just take the already parsed initializer, but instead we need to + // reparse the raw AST expression, so that `iota` gets evaluated differently for + // each spec + if (frontend.declCtx.currentDecl?.tok == 64) { + var initializerExpr = valueSpec.values.getOrNull(nameIdx) + if (initializerExpr != null) { + // Set the current initializer + frontend.declCtx.constInitializers[nameIdx] = initializerExpr + + // Set the const type + frontend.declCtx.constType = type + } else { + // Fetch expr from existing initializers + initializerExpr = frontend.declCtx.constInitializers[nameIdx] + if (initializerExpr == null) { + Util.errorWithFileLocation( + decl, + log, + "Const declaration is missing its initializer" + ) + } else { + decl.initializer = frontend.expressionHandler.handle(initializerExpr) + } + } + + type = frontend.declCtx.constType + if (type != null) { + decl.type = type + } + } + + sequence += decl } - sequence += decl + return sequence } + } + + private fun handleFuncTypeSpec( + spec: GoStandardLibrary.Ast.TypeSpec, + type: GoStandardLibrary.Ast.FuncType + ): Declaration { + // We model function types as typedef's, so that we can resolve it later + val funcType = frontend.typeOf(type) + val typedef = newTypedefDeclaration(funcType, frontend.typeOf(spec.name), rawNode = spec) + + frontend.scopeManager.addTypedef(typedef) + + return typedef + } - return sequence + private fun handleTypeDef( + spec: GoStandardLibrary.Ast.TypeSpec, + type: GoStandardLibrary.Ast.Expr + ): Declaration { + val targetType = frontend.typeOf(type) + + // We need to return either a type alias or a new type. See + // https://go.dev/ref/spec#Type_identity + return when { + // When we have an assignment, we have *identical* types. We handle them as a typedef / + // alias. + spec.assign != 0 -> { + val aliasType = frontend.typeOf(spec.name) + val typedef = newTypedefDeclaration(targetType, aliasType, rawNode = spec) + + frontend.scopeManager.addTypedef(typedef) + typedef + } + // Otherwise, we are creating a new type, which is *different*. Since Go allows to add + // methods to these kind of types, we need to create them as a record declaration. We + // use the special kind "overlay" to identity such types and put the target type in the + // list of superclasses. + else -> { + val record = newRecordDeclaration(spec.name.name, "overlay") + frontend.typeManager.registerType(record.toType()) + + // We add the underlying type as the single super class + record.superClasses = mutableListOf(targetType) + record + } + } } } diff --git a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/StatementHandler.kt b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/StatementHandler.kt index 3d84a4a80f..e94554dae9 100644 --- a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/StatementHandler.kt +++ b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/StatementHandler.kt @@ -29,6 +29,8 @@ import de.fraunhofer.aisec.cpg.graph.* import de.fraunhofer.aisec.cpg.graph.declarations.DeclarationSequence import de.fraunhofer.aisec.cpg.graph.statements.* import de.fraunhofer.aisec.cpg.graph.statements.expressions.* +import de.fraunhofer.aisec.cpg.graph.types.HasType +import de.fraunhofer.aisec.cpg.graph.types.Type class StatementHandler(frontend: GoLanguageFrontend) : GoHandler(::ProblemExpression, frontend) { @@ -51,7 +53,9 @@ class StatementHandler(frontend: GoLanguageFrontend) : is GoStandardLibrary.Ast.LabeledStmt -> handleLabeledStmt(stmt) is GoStandardLibrary.Ast.RangeStmt -> handleRangeStmt(stmt) is GoStandardLibrary.Ast.ReturnStmt -> handleReturnStmt(stmt) + is GoStandardLibrary.Ast.SendStmt -> handleSendStmt(stmt) is GoStandardLibrary.Ast.SwitchStmt -> handleSwitchStmt(stmt) + is GoStandardLibrary.Ast.TypeSwitchStmt -> handleTypeSwitchStmt(stmt) else -> handleNotSupported(stmt, stmt.goType) } } @@ -114,31 +118,100 @@ class StatementHandler(frontend: GoLanguageFrontend) : return compound } - private fun handleCaseClause(caseClause: GoStandardLibrary.Ast.CaseClause): Statement { + private fun handleCaseClause( + caseClause: GoStandardLibrary.Ast.CaseClause, + typeSwitchLhs: Node? = null, + typeSwitchRhs: Expression? = null, + ): Statement { + val isTypeSwitch = typeSwitchRhs != null + val case = if (caseClause.list.isEmpty()) { newDefaultStatement(rawNode = caseClause) } else { val case = newCaseStatement(rawNode = caseClause) - case.caseExpression = frontend.expressionHandler.handle(caseClause.list[0]) + if (isTypeSwitch) { + // If this case is within a type switch, we want to wrap the case expression in + // a TypeExpression + val type = frontend.typeOf(caseClause.list[0]) + case.caseExpression = newTypeExpression(type.name, type) + } else { + case.caseExpression = frontend.expressionHandler.handle(caseClause.list[0]) + } case } // We need to find the current block / scope and add the statements to it - val block = frontend.scopeManager.currentBlock + val currentBlock = frontend.scopeManager.currentBlock - if (block == null) { + if (currentBlock == null) { log.error("could not find block to add case clauses") return newProblemExpression("could not find block to add case clauses") } // Add the case statement - block += case + currentBlock += case + + // Wrap everything inside the case in a block statement, if this is a type-switch, so that + // we can re-declare the variable locally in the block. + val block = + if (isTypeSwitch) { + newBlock() + } else { + null + } + + block?.let { frontend.scopeManager.enterScope(it) } + + // TODO(oxisto): This variable is not yet resolvable + if (isTypeSwitch && typeSwitchRhs != null && typeSwitchLhs != null) { + val stmt = newDeclarationStatement() + stmt.isImplicit = true + + val decl = newVariableDeclaration(typeSwitchLhs.name) + if (case is CaseStatement) { + decl.type = (case.caseExpression as? TypeExpression)?.type ?: unknownType() + } else { + // We need to work with type listeners here because they might not have their type + // yet + typeSwitchRhs.registerTypeObserver( + object : HasType.TypeObserver { + override fun typeChanged(newType: Type, src: HasType) { + decl.type = newType + } + + override fun assignedTypeChanged(assignedTypes: Set, src: HasType) { + // Nothing to do + } + } + ) + } + decl.initializer = typeSwitchRhs + + // Add the variable to the declaration statement as well as to the current scope (aka + // our block wrapper) + stmt.addToPropertyEdgeDeclaration(decl) + frontend.scopeManager.addDeclaration(decl) + + if (block != null) { + block += stmt + } + } for (s in caseClause.body) { - block += handle(s) + if (block != null) { + block += handle(s) + } else { + currentBlock += handle(s) + } } + if (block != null) { + currentBlock += block + } + + block?.let { frontend.scopeManager.leaveScope(it) } + // this is a little trick, to not add the case statement in handleStmt because we added it // already. otherwise, the order is screwed up. return case @@ -285,9 +358,7 @@ class StatementHandler(frontend: GoLanguageFrontend) : val expr = frontend.expressionHandler.handle(results[0]) // TODO: parse more than one result expression - if (expr != null) { - `return`.returnValue = expr - } + `return`.returnValue = expr } else { // TODO: connect result statement to result variables } @@ -295,6 +366,14 @@ class StatementHandler(frontend: GoLanguageFrontend) : return `return` } + private fun handleSendStmt(sendStmt: GoStandardLibrary.Ast.SendStmt): BinaryOperator { + val op = newBinaryOperator("<-", rawNode = sendStmt) + op.lhs = frontend.expressionHandler.handle(sendStmt.chan) + op.rhs = frontend.expressionHandler.handle(sendStmt.value) + + return op + } + private fun handleSwitchStmt(switchStmt: GoStandardLibrary.Ast.SwitchStmt): Statement { val switch = newSwitchStatement(rawNode = switchStmt) @@ -306,19 +385,46 @@ class StatementHandler(frontend: GoLanguageFrontend) : val block = handle(switchStmt.body) as? Block ?: return newProblemExpression("missing switch body") - // Because of the way we parse the statements, the case statement turns out to be the last - // statement. However, we need it to be the first statement, so we need to switch first and - // last items - /*val statements = block.statements.toMutableList() - val tmp = statements.first() - statements[0] = block.statements.last() - statements[(statements.size - 1).coerceAtLeast(0)] = tmp - block.statements = statements*/ - switch.statement = block frontend.scopeManager.leaveScope(switch) return switch } + + private fun handleTypeSwitchStmt( + typeSwitchStmt: GoStandardLibrary.Ast.TypeSwitchStmt + ): SwitchStatement { + val switch = newSwitchStatement(rawNode = typeSwitchStmt) + + frontend.scopeManager.enterScope(switch) + + typeSwitchStmt.init?.let { switch.initializerStatement = handle(it) } + + val assign = frontend.statementHandler.handle(typeSwitchStmt.assign) + val (lhs, rhs) = + if (assign is AssignExpression) { + val rhs = assign.rhs.singleOrNull() + switch.selector = rhs + Pair(assign.lhs.singleOrNull(), (rhs as? UnaryOperator)?.input) + } else { + Pair(null, null) + } + + val body = newBlock(rawNode = typeSwitchStmt.body) + + frontend.scopeManager.enterScope(body) + + for (c in typeSwitchStmt.body.list.filterIsInstance()) { + handleCaseClause(c, lhs, rhs) + } + + frontend.scopeManager.leaveScope(body) + + switch.statement = body + + frontend.scopeManager.leaveScope(switch) + + return switch + } } diff --git a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/GoEvaluationOrderGraphPass.kt b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/GoEvaluationOrderGraphPass.kt index e519e15f30..2d99d51102 100644 --- a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/GoEvaluationOrderGraphPass.kt +++ b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/GoEvaluationOrderGraphPass.kt @@ -28,6 +28,8 @@ package de.fraunhofer.aisec.cpg.passes import de.fraunhofer.aisec.cpg.TranslationContext import de.fraunhofer.aisec.cpg.frontends.golang.GoLanguage import de.fraunhofer.aisec.cpg.graph.declarations.FunctionDeclaration +import de.fraunhofer.aisec.cpg.graph.declarations.NamespaceDeclaration +import de.fraunhofer.aisec.cpg.graph.declarations.RecordDeclaration import de.fraunhofer.aisec.cpg.graph.followNextEOGEdgesUntilHit import de.fraunhofer.aisec.cpg.graph.statements.ReturnStatement import de.fraunhofer.aisec.cpg.graph.statements.expressions.CallExpression @@ -80,6 +82,23 @@ class GoEvaluationOrderGraphPass(ctx: TranslationContext) : EvaluationOrderGraph } } + /** + * We need to intentionally override [handleRecordDeclaration] to NOT create the EOG for its + * children, e.g., [RecordDeclaration.methods]. The reason for this is that Go only has external + * methods declarations, and we do a little cheat by adding the methods both to the namespace of + * the current file and to the [RecordDeclaration.methods]. + * + * But, due to this, the original [EvaluationOrderGraphPass] would create the EOG for methods + * twice, once for the object in the [RecordDeclaration] and once for the declaration inside the + * [NamespaceDeclaration] of the current file. + */ + override fun handleRecordDeclaration(node: RecordDeclaration) { + scopeManager.enterScope(node) + handleStatementHolder(node) + currentPredecessors.clear() + scopeManager.leaveScope(node) + } + override fun handleFunctionDeclaration(node: FunctionDeclaration) { // First, call the regular EOG handler super.handleFunctionDeclaration(node) diff --git a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/GoExtraPass.kt b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/GoExtraPass.kt index f9c20d952e..ea3e154cf2 100644 --- a/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/GoExtraPass.kt +++ b/cpg-language-go/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/GoExtraPass.kt @@ -27,21 +27,19 @@ package de.fraunhofer.aisec.cpg.passes import de.fraunhofer.aisec.cpg.TranslationContext import de.fraunhofer.aisec.cpg.frontends.golang.GoLanguage +import de.fraunhofer.aisec.cpg.frontends.golang.funcTypeName +import de.fraunhofer.aisec.cpg.frontends.golang.isOverlay +import de.fraunhofer.aisec.cpg.frontends.golang.underlyingType import de.fraunhofer.aisec.cpg.graph.* import de.fraunhofer.aisec.cpg.graph.declarations.IncludeDeclaration import de.fraunhofer.aisec.cpg.graph.declarations.NamespaceDeclaration +import de.fraunhofer.aisec.cpg.graph.declarations.TranslationUnitDeclaration import de.fraunhofer.aisec.cpg.graph.declarations.VariableDeclaration import de.fraunhofer.aisec.cpg.graph.scopes.Scope import de.fraunhofer.aisec.cpg.graph.statements.DeclarationStatement import de.fraunhofer.aisec.cpg.graph.statements.ForEachStatement -import de.fraunhofer.aisec.cpg.graph.statements.expressions.AssignExpression -import de.fraunhofer.aisec.cpg.graph.statements.expressions.CallExpression -import de.fraunhofer.aisec.cpg.graph.statements.expressions.CastExpression -import de.fraunhofer.aisec.cpg.graph.statements.expressions.Reference -import de.fraunhofer.aisec.cpg.graph.types.HasType -import de.fraunhofer.aisec.cpg.graph.types.PointerType -import de.fraunhofer.aisec.cpg.graph.types.Type -import de.fraunhofer.aisec.cpg.graph.types.UnknownType +import de.fraunhofer.aisec.cpg.graph.statements.expressions.* +import de.fraunhofer.aisec.cpg.graph.types.* import de.fraunhofer.aisec.cpg.helpers.SubgraphWalker import de.fraunhofer.aisec.cpg.passes.inference.startInference import de.fraunhofer.aisec.cpg.passes.order.ExecuteBefore @@ -101,6 +99,13 @@ import de.fraunhofer.aisec.cpg.passes.order.ExecuteBefore * they are compatible. Because types in the same package can be defined in multiple files, we * cannot decide during the frontend run. Therefore, we need to execute this pass before the * [CallResolver] and convert certain [CallExpression] nodes into a [CastExpression]. + * + * ## Adjust Names of Keys in Key Value Expressions to FQN + * + * This pass also adjusts the names of keys in a [KeyValueExpression], which is part of an + * [InitializerListExpression] to a fully-qualified name that contains the name of the [ObjectType] + * that the expression is creating. This way we can resolve the static references to the field to + * the actual field. */ @ExecuteBefore(VariableUsageResolver::class) @ExecuteBefore(CallResolver::class) @@ -111,6 +116,11 @@ class GoExtraPass(ctx: TranslationContext) : ComponentPass(ctx), ScopeProvider { get() = scopeManager.currentScope override fun accept(component: Component) { + // Add built-int functions, but only if one of the components contains a GoLanguage + if (component.translationUnits.any { it.language is GoLanguage }) { + component.translationUnits += addBuiltIn() + } + val walker = SubgraphWalker.ScopedWalker(scopeManager) walker.registerHandler { _, parent, node -> when (node) { @@ -118,6 +128,7 @@ class GoExtraPass(ctx: TranslationContext) : ComponentPass(ctx), ScopeProvider { is IncludeDeclaration -> handleInclude(node) is AssignExpression -> handleAssign(node) is ForEachStatement -> handleForEachStatement(node) + is InitializerListExpression -> handleInitializerListExpression(node) } } @@ -126,6 +137,106 @@ class GoExtraPass(ctx: TranslationContext) : ComponentPass(ctx), ScopeProvider { } } + private fun addBuiltIn(): TranslationUnitDeclaration { + val builtin = newTranslationUnitDeclaration("builtin.go") + scopeManager.resetToGlobal(builtin) + + val len = newFunctionDeclaration("len", localNameOnly = true) + len.parameters = listOf(newParameterDeclaration("v", GoLanguage().autoType())) + len.type = + typeManager.registerType( + FunctionType(funcTypeName(len.signatureTypes, len.returnTypes)) + ) + scopeManager.addDeclaration(len) + + /** + * ```go + * func append(slice []Type, elems ...Type) []Type + * ``` + */ + val append = newFunctionDeclaration("append", localNameOnly = true) + append.parameters = + listOf( + newParameterDeclaration("slice", GoLanguage().autoType().array()), + newParameterDeclaration("elems", GoLanguage().autoType(), variadic = true), + ) + append.returnTypes = listOf(GoLanguage().autoType().array()) + append.type = + typeManager.registerType( + FunctionType(funcTypeName(append.signatureTypes, append.returnTypes)) + ) + scopeManager.addDeclaration(append) + + val error = newRecordDeclaration("error", "interface") + scopeManager.enterScope(error) + + val errorFunc = newMethodDeclaration("Error", recordDeclaration = error) + errorFunc.returnTypes = listOf(GoLanguage().primitiveType("string")) + errorFunc.type = + typeManager.registerType( + FunctionType(funcTypeName(errorFunc.signatureTypes, errorFunc.returnTypes)) + ) + scopeManager.addDeclaration(errorFunc) + + scopeManager.leaveScope(error) + + return builtin + } + + /** + * handleInitializerListExpression changes the references of keys in a [KeyValueExpression] to + * include the object it is creating as a parent name. + */ + private fun handleInitializerListExpression(node: InitializerListExpression) { + var type: Type? = node.type + + // If our type is an "overlay", we need to look for the underlying type + type = + if (type.isOverlay) { + type.underlyingType + } else { + type + } + + // The type of a "inner" composite literal can be omitted if the outer one is creating + // an array type. In this case, we need to set the type manually because the type for + // the "inner" one is empty. + // Example code: + // ```go + // var a = []*MyObject{ + // { + // Name: "a", + // }, + // { + // Name: "b", + // } + // } + if (type is PointerType && type.isArray) { + for (init in node.initializers) { + if (init is InitializerListExpression) { + init.type = type.elementType + } + } + } + + // We are not interested in arrays and maps, but only the "inner" single-object expressions + if ( + type is UnknownType || + (type is PointerType && type.isArray) || + node.type.name.localName == "map" + ) { + return + } + + for (keyValue in node.initializers.filterIsInstance()) { + val key = keyValue.key + if (key is Reference) { + key.name = Name(key.name.localName, node.type.root.name) + key.isStaticAccess = true + } + } + } + /** * handleForEachStatement adds a [HasType.TypeObserver] to the [ForEachStatement.iterable] of an * [ForEachStatement] in order to determine the types used in [ForEachStatement.variable] (index @@ -173,16 +284,23 @@ class GoExtraPass(ctx: TranslationContext) : ComponentPass(ctx), ScopeProvider { } // Loop through the target variables (left-hand side) - for (expr in assign.lhs) { + for ((idx, expr) in assign.lhs.withIndex()) { if (expr is Reference) { // And try to resolve it val ref = scopeManager.resolveReference(expr) if (ref == null) { - // We need to implicitly declare it, if its not declared before. + // We need to implicitly declare it, if it's not declared before. val decl = newVariableDeclaration(expr.name, expr.autoType()) decl.location = expr.location decl.isImplicit = true - decl.initializer = assign.findValue(expr) + + // We cannot assign an initializer here because this will lead to duplicate + // DFG edges, but we need to propagate the type information + if (assign.rhs.size < assign.lhs.size && assign.rhs.size == 1) { + assign.rhs[0].registerTypeObserver(InitializerTypePropagation(decl, idx)) + } else { + assign.rhs[idx].registerTypeObserver(InitializerTypePropagation(decl)) + } assign.declarations += decl @@ -230,13 +348,20 @@ class GoExtraPass(ctx: TranslationContext) : ComponentPass(ctx), ScopeProvider { // We need to check, whether the "callee" refers to a type and if yes, convert it into a // cast expression. And this is only really necessary, if the function call has a single // argument. - val callee = call.callee - if (parent != null && callee is Reference && call.arguments.size == 1) { + var callee = call.callee + if (parent != null && callee != null && call.arguments.size == 1) { val language = parent.language ?: GoLanguage() + var pointer = false + // If the argument is a UnaryOperator, unwrap them + if (callee is UnaryOperator && callee.operatorCode == "*") { + pointer = true + callee = callee.input + } + // First, check if this is a built-in type if (language.builtInTypes.contains(callee.name.toString())) { - replaceCallWithCast(callee.name.toString(), parent, call) + replaceCallWithCast(callee.name.toString(), parent, call, false) } else { // If not, then this could still refer to an existing type. We need to make sure // that we take the current namespace into account @@ -248,7 +373,7 @@ class GoExtraPass(ctx: TranslationContext) : ComponentPass(ctx), ScopeProvider { } if (typeManager.typeExists(fqn.toString())) { - replaceCallWithCast(fqn, parent, call) + replaceCallWithCast(fqn, parent, call, pointer) } } } @@ -258,10 +383,16 @@ class GoExtraPass(ctx: TranslationContext) : ComponentPass(ctx), ScopeProvider { typeName: CharSequence, parent: Node, call: CallExpression, + pointer: Boolean, ) { val cast = parent.newCastExpression(call.code) cast.location = call.location - cast.castType = call.objectType(typeName) + cast.castType = + if (pointer) { + call.objectType(typeName).pointer() + } else { + call.objectType(typeName) + } cast.expression = call.arguments.single() if (parent !is ArgumentHolder) { @@ -284,4 +415,19 @@ class GoExtraPass(ctx: TranslationContext) : ComponentPass(ctx), ScopeProvider { override fun cleanup() { // Nothing to do } + + class InitializerTypePropagation(private var decl: HasType, private var tupleIdx: Int = -1) : + HasType.TypeObserver { + override fun typeChanged(newType: Type, src: HasType) { + if (newType is TupleType && tupleIdx != -1) { + decl.type = newType.types.getOrElse(tupleIdx) { decl.unknownType() } + } else { + decl.type = newType + } + } + + override fun assignedTypeChanged(assignedTypes: Set, src: HasType) { + // TODO + } + } } diff --git a/cpg-language-go/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/BuildConstraintsTest.kt b/cpg-language-go/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/BuildConstraintsTest.kt new file mode 100644 index 0000000000..814456b487 --- /dev/null +++ b/cpg-language-go/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/BuildConstraintsTest.kt @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2023, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.frontends.golang + +import kotlin.test.* + +class BuildConstraintsTest { + + @Test + fun testFromString() { + val string = "darwin && !amd64" + + val root = BuildConstraintExpression.fromString(string) + assertNotNull(root) + + var expr = root + assertIs(expr) + assertEquals("&&", expr.operatorCode) + + val lhs = expr.lhs + assertIs(lhs) + assertEquals("darwin", lhs.tag) + + val rhs = expr.rhs + assertIs(rhs) + assertEquals("!", rhs.operatorCode) + + expr = rhs.expr + assertIs(expr) + assertEquals("amd64", expr.tag) + + val darwinAmd64 = setOf("darwin", "amd64") + assertFalse(root.evaluate(darwinAmd64)) + + val darwinArm64 = setOf("darwin", "arm64") + assertTrue(root.evaluate(darwinArm64)) + } +} diff --git a/cpg-language-go/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/DeclarationTest.kt b/cpg-language-go/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/DeclarationTest.kt index ff07d6aa1a..62a1cb78db 100644 --- a/cpg-language-go/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/DeclarationTest.kt +++ b/cpg-language-go/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/DeclarationTest.kt @@ -26,15 +26,17 @@ package de.fraunhofer.aisec.cpg.frontends.golang import de.fraunhofer.aisec.cpg.TestUtils +import de.fraunhofer.aisec.cpg.TestUtils.assertInvokes +import de.fraunhofer.aisec.cpg.TestUtils.assertRefersTo import de.fraunhofer.aisec.cpg.assertFullName import de.fraunhofer.aisec.cpg.assertLiteralValue +import de.fraunhofer.aisec.cpg.assertLocalName import de.fraunhofer.aisec.cpg.graph.* import de.fraunhofer.aisec.cpg.graph.byNameOrNull -import de.fraunhofer.aisec.cpg.graph.declarations.FunctionDeclaration -import de.fraunhofer.aisec.cpg.graph.declarations.MethodDeclaration -import de.fraunhofer.aisec.cpg.graph.declarations.NamespaceDeclaration -import de.fraunhofer.aisec.cpg.graph.declarations.RecordDeclaration -import de.fraunhofer.aisec.cpg.graph.statements.expressions.Reference +import de.fraunhofer.aisec.cpg.graph.declarations.* +import de.fraunhofer.aisec.cpg.graph.statements.ReturnStatement +import de.fraunhofer.aisec.cpg.graph.statements.expressions.* +import de.fraunhofer.aisec.cpg.graph.types.ObjectType import de.fraunhofer.aisec.cpg.graph.variables import java.nio.file.Path import kotlin.test.* @@ -90,6 +92,89 @@ class DeclarationTest { assertFullName("", param) } + @Test + fun testStruct() { + val topLevel = Path.of("src", "test", "resources", "golang") + val tu = + TestUtils.analyzeAndGetFirstTU( + listOf(topLevel.resolve("struct.go").toFile()), + topLevel, + true + ) { + it.registerLanguage() + } + assertNotNull(tu) + + val p = tu.namespaces["p"] + assertNotNull(p) + + val myStruct = p.records["MyStruct"] + assertNotNull(myStruct) + assertEquals("struct", myStruct.kind) + + val fields = myStruct.fields + assertEquals( + listOf("MyField", "OtherStruct", "EvenAnotherStruct"), + fields.map { it.name.localName } + ) + + var methods = myStruct.methods + + var myFunc = methods.firstOrNull() + assertNotNull(myFunc) + assertFullName("p.MyStruct.MyFunc", myFunc) + + val myField = fields.firstOrNull() + assertNotNull(myField) + + assertLocalName("MyField", myField) + assertEquals(tu.primitiveType("int"), myField.type) + + val myInterface = p.records["p.MyInterface"] + assertNotNull(myInterface) + assertEquals("interface", myInterface.kind) + + methods = myInterface.methods + + assertEquals(1, methods.size) + + myFunc = methods.first() + + assertLocalName("MyFunc", myFunc) + assertLocalName("func() string", myFunc.type) + + val newMyStruct = p.functions["NewMyStruct"] + assertNotNull(newMyStruct) + + val body = newMyStruct.body as? Block + assertNotNull(body) + + val `return` = body.statements.first() as? ReturnStatement + assertNotNull(`return`) + + val returnValue = `return`.returnValue as? UnaryOperator + assertNotNull(returnValue) + + val s = p.variables["p.s"] + assertNotNull(s) + + val type = s.type + assertIs(type) + + val record = type.recordDeclaration + assertNotNull(record) + + val init = s.initializer + assertIs(init) + + val keyValue = init.initializers(0) + assertNotNull(keyValue) + + val key = keyValue.key + assertNotNull(key) + assertRefersTo(key, record.fields["field"]) + } + @Test fun testEmbeddedInterface() { val topLevel = Path.of("src", "test", "resources", "golang") @@ -134,8 +219,8 @@ class DeclarationTest { val main = tu.functions["main.main"] assertNotNull(main) - // We should have 7 variables (a, b, c, d, e, f, g) - assertEquals(7, tu.variables.size) + // We should have 10 variables (a, b, c, d, (e,f), e, f, g, h, i) + assertEquals(10, main.variables.size) // Four should have (literal) initializers val a = main.variables["a"] @@ -150,13 +235,24 @@ class DeclarationTest { val d = main.variables["d"] assertLiteralValue(4, d?.initializer) + val e = main.variables["e"] + assertNotNull(e) + // e does not have a direct initializer, since it is initialized through the tuple + // declaration (e,f) + assertNull(e.initializer) + + // The tuple (e,f) does have an initializer + val ef = main.allChildren { it.name.toString() == "(e,f)" }.firstOrNull() + assertNotNull(ef) + assertIs(ef.initializer) + // The next two variables are using a short assignment, therefore they do not have an // initializer, but we can use the firstAssignment function - val e = main.variables["e"] - assertLiteralValue(5, e?.firstAssignment) + val g = main.variables["g"] + assertLiteralValue(5, g?.firstAssignment) - val f = main.variables["f"] - assertLiteralValue(6, f?.firstAssignment) + val h = main.variables["h"] + assertLiteralValue(6, h?.firstAssignment) // And they should all be connected to the arguments of the Printf call val printf = main.calls["Printf"] @@ -167,8 +263,8 @@ class DeclarationTest { assertNotNull(ref.refersTo) } - // We have eight assignments in total (6 initializers + 2 assign expressions) - assertEquals(8, tu.assignments.size) + // We have eight assignments in total (7 initializers + 2 assign expressions) + assertEquals(9, tu.assignments.size) } @Test @@ -190,4 +286,109 @@ class DeclarationTest { val myInterface = tu.records["MyInterface"] assertNotNull(myInterface) } + + @Test + fun testConst() { + val topLevel = Path.of("src", "test", "resources", "golang") + val tu = + TestUtils.analyzeAndGetFirstTU( + listOf(topLevel.resolve("const.go").toFile()), + topLevel, + true + ) { + it.registerLanguage() + } + assertNotNull(tu) + + with(tu) { + val values = + mapOf( + "one" to Pair(1, objectType("p.custom")), + "oneAsWell" to Pair(1, objectType("p.custom")), + "two" to Pair(2, primitiveType("int")), + "three" to Pair(3, primitiveType("int")), + "four" to Pair(4, primitiveType("int")), + "tenAsWell" to Pair(10, primitiveType("int")), + "five" to Pair(5, primitiveType("int")), + "fiveAsWell" to Pair(5, primitiveType("int")), + "six" to Pair(6, primitiveType("int")), + "fivehundred" to Pair(500, primitiveType("int")), + "sixhundred" to Pair(600, primitiveType("int")), + "onehundredandfive" to Pair(105, primitiveType("int")), + ) + values.forEach { + val variable = tu.variables[it.key] + assertNotNull(variable, "variable \"${it.key}\" not found") + assertEquals(it.value.first, variable.evaluate(), "${it.key} does not match") + assertEquals(it.value.second, variable.type, "${it.key} has the wrong type") + } + } + } + + @Test + fun testImportAlias() { + val stdLib = Path.of("src", "test", "resources", "golang-std") + val topLevel = Path.of("src", "test", "resources", "golang") + val result = + TestUtils.analyze( + listOf( + topLevel.resolve("importalias.go").toFile(), + stdLib.resolve("fmt").toFile(), + ), + topLevel, + true + ) { + it.registerLanguage() + it.includePath(stdLib) + } + assertNotNull(result) + + val printf = result.functions["fmt.Printf"] + assertNotNull(printf) + + val callPrintf = result.calls["fmtother.Printf"] + assertNotNull(callPrintf) + assertInvokes(callPrintf, printf) + + val expr = result.allChildren().firstOrNull() + assertNotNull(expr) + + val fmt = result.variables["fmt"] + assertNotNull(fmt) + + val base = expr.base + assertIs(base) + assertRefersTo(base, fmt) + } + + @Test + fun testFuncOptions() { + val topLevel = Path.of("src", "test", "resources", "golang", "options") + val result = + TestUtils.analyze( + listOf( + topLevel.resolve("srv_option.go").toFile(), + topLevel.resolve("srv.go").toFile(), + ), + topLevel, + true + ) { + it.registerLanguage() + } + assertNotNull(result) + + val inner = result.records["inner"] + assertNotNull(inner) + + val field = inner.fields["field"] + assertNotNull(field) + + val assign = result.assignments.firstOrNull() + assertNotNull(assign) + + val mce = assign.target + assertNotNull(mce) + assertIs(mce) + assertRefersTo(mce, field) + } } diff --git a/cpg-language-go/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/ExpressionTest.kt b/cpg-language-go/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/ExpressionTest.kt index 3bcdab33be..a532155c0c 100644 --- a/cpg-language-go/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/ExpressionTest.kt +++ b/cpg-language-go/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/ExpressionTest.kt @@ -26,6 +26,7 @@ package de.fraunhofer.aisec.cpg.frontends.golang import de.fraunhofer.aisec.cpg.TestUtils +import de.fraunhofer.aisec.cpg.TestUtils.assertRefersTo import de.fraunhofer.aisec.cpg.assertFullName import de.fraunhofer.aisec.cpg.assertLiteralValue import de.fraunhofer.aisec.cpg.assertLocalName @@ -100,7 +101,7 @@ class ExpressionTest { // [:1] var slice = assertIs( - assertIs(b.initializer).subscriptExpression + assertIs(b.firstAssignment).subscriptExpression ) assertNull(slice.floor) assertLiteralValue(1, slice.ceiling) @@ -111,7 +112,7 @@ class ExpressionTest { assertLocalName("int[]", c.type) // [1:] - slice = assertIs(assertIs(c.initializer).subscriptExpression) + slice = assertIs(assertIs(c.firstAssignment).subscriptExpression) assertLiteralValue(1, slice.floor) assertNull(slice.ceiling) assertNull(slice.third) @@ -121,7 +122,7 @@ class ExpressionTest { assertLocalName("int[]", d.type) // [0:1] - slice = assertIs(assertIs(d.initializer).subscriptExpression) + slice = assertIs(assertIs(d.firstAssignment).subscriptExpression) assertLiteralValue(0, slice.floor) assertLiteralValue(1, slice.ceiling) assertNull(slice.third) @@ -131,9 +132,45 @@ class ExpressionTest { assertLocalName("int[]", e.type) // [0:1:1] - slice = assertIs(assertIs(e.initializer).subscriptExpression) + slice = assertIs(assertIs(e.firstAssignment).subscriptExpression) assertLiteralValue(0, slice.floor) assertLiteralValue(1, slice.ceiling) assertLiteralValue(1, slice.third) } + + @Test + fun testSendStmt() { + val topLevel = Path.of("src", "test", "resources", "golang") + val tu = + TestUtils.analyzeAndGetFirstTU( + listOf(topLevel.resolve("chan.go").toFile()), + topLevel, + true + ) { + it.registerLanguage() + } + assertNotNull(tu) + + with(tu) { + val main = tu.functions["main"] + assertNotNull(main) + + val v = main.variables["v"] + assertNotNull(v) + assertEquals(primitiveType("int"), v.type) + + val ch = main.variables["ch"] + assertNotNull(ch) + assertEquals(objectType("chan", generics = listOf(primitiveType("int"))), ch.type) + + val binOp = main.bodyOrNull() + assertNotNull(binOp) + assertRefersTo(binOp.lhs, ch) + assertRefersTo(binOp.rhs, v) + + val unaryOp = main.bodyOrNull() + assertNotNull(unaryOp) + assertRefersTo(unaryOp.input, ch) + } + } } diff --git a/cpg-language-go/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoLanguageFrontendTest.kt b/cpg-language-go/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoLanguageFrontendTest.kt index c778028686..66ae52b06b 100644 --- a/cpg-language-go/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoLanguageFrontendTest.kt +++ b/cpg-language-go/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoLanguageFrontendTest.kt @@ -28,7 +28,10 @@ package de.fraunhofer.aisec.cpg.frontends.golang import de.fraunhofer.aisec.cpg.BaseTest import de.fraunhofer.aisec.cpg.TestUtils.analyze import de.fraunhofer.aisec.cpg.TestUtils.analyzeAndGetFirstTU +import de.fraunhofer.aisec.cpg.TestUtils.assertInvokes +import de.fraunhofer.aisec.cpg.TestUtils.assertRefersTo import de.fraunhofer.aisec.cpg.assertFullName +import de.fraunhofer.aisec.cpg.assertLiteralValue import de.fraunhofer.aisec.cpg.assertLocalName import de.fraunhofer.aisec.cpg.graph.* import de.fraunhofer.aisec.cpg.graph.declarations.FunctionDeclaration @@ -38,6 +41,9 @@ import de.fraunhofer.aisec.cpg.graph.declarations.VariableDeclaration import de.fraunhofer.aisec.cpg.graph.statements.* import de.fraunhofer.aisec.cpg.graph.statements.expressions.* import de.fraunhofer.aisec.cpg.graph.types.FunctionType +import de.fraunhofer.aisec.cpg.graph.types.ObjectType +import de.fraunhofer.aisec.cpg.graph.types.PointerType +import java.io.File import java.nio.file.Path import kotlin.test.* @@ -61,16 +67,13 @@ class GoLanguageFrontendTest : BaseTest() { val message = main.variables["message"] assertNotNull(message) - val map = - assertIs( - assertIs(message.firstAssignment).arguments.firstOrNull() - ) + val map = assertIs(message.firstAssignment) assertNotNull(map) val nameEntry = map.initializers.firstOrNull() as? KeyValueExpression assertNotNull(nameEntry) - assertLocalName("string[]", (nameEntry.value as? ConstructExpression)?.type) + assertLocalName("string[]", (nameEntry.value as? InitializerListExpression)?.type) } @Test @@ -96,7 +99,7 @@ class GoLanguageFrontendTest : BaseTest() { val path = data.firstAssignment?.followPrevDFG { it is KeyValueExpression } assertNotNull(path) - assertEquals(3, path.size) + assertEquals(2, path.size) } @Test @@ -178,11 +181,20 @@ class GoLanguageFrontendTest : BaseTest() { @Test fun testLiteral() { val topLevel = Path.of("src", "test", "resources", "golang") - val tu = - analyzeAndGetFirstTU(listOf(topLevel.resolve("literal.go").toFile()), topLevel, true) { + val result = + analyze( + listOf( + topLevel.resolve("literal.go").toFile(), + topLevel.resolve("submodule/const.go").toFile(), + ), + topLevel, + true + ) { it.registerLanguage() } + assertNotNull(result) + val tu = result.components["application"]?.translationUnits?.firstOrNull() assertNotNull(tu) val p = tu.namespaces["p"] @@ -230,6 +242,82 @@ class GoLanguageFrontendTest : BaseTest() { assertFullName("", func) assertEquals(1, func.parameters.size) assertEquals(1, func.returnTypes.size) + + val o = p.variables["o"] + assertNotNull(o) + assertLocalName("MyStruct[]", o.type) + + val myStruct = tu.records["MyStruct"] + assertNotNull(myStruct) + + val field = myStruct.fields["Field"] + assertNotNull(field) + + var composite = + (o.initializer as? InitializerListExpression)?.initializers( + 0 + ) + assertNotNull(composite) + assertIs(composite) + assertIs(composite.type) + assertLocalName("MyStruct", composite.type) + + var keyValue = composite.initializers(0) + assertNotNull(keyValue) + assertLocalName("Field", keyValue.key) + assertRefersTo(keyValue.key, field) + assertLiteralValue(10, keyValue.value) + + val o3 = p.variables["o3"] + assertNotNull(o3) + assertLocalName("MyStruct[]", o.type) + + composite = + (o.initializer as? InitializerListExpression)?.initializers( + 0 + ) + assertNotNull(composite) + assertIs(composite) + assertIs(composite.type) + assertLocalName("MyStruct", composite.type) + + keyValue = composite.initializers(0) + assertNotNull(keyValue) + assertLocalName("Field", keyValue.key) + assertRefersTo(keyValue.key, field) + assertLiteralValue(10, keyValue.value) + + val rr = tu.variables["rr"] + assertNotNull(rr) + + var init = rr.initializer + assertIs(init) + + keyValue = init.initializers(0) + assertNotNull(keyValue) + + var key = keyValue.key + assertNotNull(key) + + var zero = result.variables["submodule.Zero"] + assertNotNull(zero) + assertRefersTo(key, zero) + + val mapr = tu.variables["mapr"] + assertNotNull(mapr) + + init = mapr.initializer + assertIs(init) + + keyValue = init.initializers(0) + assertNotNull(keyValue) + + key = keyValue.key + assertNotNull(key) + + zero = result.variables["submodule.Zero"] + assertNotNull(zero) + assertRefersTo(key, zero) } @Test @@ -248,22 +336,24 @@ class GoLanguageFrontendTest : BaseTest() { val main = p.functions["main"] assertNotNull(main) - var type = main.type as? FunctionType + var type = main.type + assertIs(type) assertNotNull(type) assertLocalName("func()", type) assertEquals(0, type.parameters.size) assertEquals(0, type.returnTypes.size) - val myTest = p.functions["myTest"] - assertNotNull(myTest) - assertEquals(1, myTest.parameters.size) - assertEquals(2, myTest.returnTypes.size) + val funcA = p.functions["funcA"] + assertNotNull(funcA) + assertEquals(1, funcA.parameters.size) + assertEquals(2, funcA.returnTypes.size) - type = myTest.type as? FunctionType + type = funcA.type + assertIs(type) assertNotNull(type) assertLocalName("func(string) (int, error)", type) - assertEquals(myTest.parameters.size, type.parameters.size) - assertEquals(myTest.returnTypes.size, type.returnTypes.size) + assertEquals(funcA.parameters.size, type.parameters.size) + assertEquals(funcA.returnTypes.size, type.returnTypes.size) assertEquals(listOf("int", "error"), type.returnTypes.map { it.name.localName }) var body = main.body as? Block @@ -272,17 +362,17 @@ class GoLanguageFrontendTest : BaseTest() { var callExpression = body.calls.firstOrNull() assertNotNull(callExpression) - assertLocalName("myTest", callExpression) - assertEquals(myTest, callExpression.invokes.iterator().next()) + assertLocalName("funcA", callExpression) + assertEquals(funcA, callExpression.invokes.iterator().next()) - val s = myTest.parameters.first() + val s = funcA.parameters.first() assertNotNull(s) assertLocalName("s", s) assertEquals(tu.primitiveType("string"), s.type) - assertLocalName("myTest", myTest) + assertLocalName("funcA", funcA) - body = myTest.body as? Block + body = funcA.body as? Block assertNotNull(body) callExpression = body.statements.first() as? CallExpression @@ -329,70 +419,34 @@ class GoLanguageFrontendTest : BaseTest() { assertNotNull(err) assertLocalName("error", err.type) - } - @Test - fun testStruct() { - val topLevel = Path.of("src", "test", "resources", "golang") - val tu = - analyzeAndGetFirstTU(listOf(topLevel.resolve("struct.go").toFile()), topLevel, true) { - it.registerLanguage() - } - - assertNotNull(tu) - - val p = tu.getDeclarationsByName("p", NamespaceDeclaration::class.java).iterator().next() - - val myStruct = - p.getDeclarationsByName("p.MyStruct", RecordDeclaration::class.java).iterator().next() - - assertNotNull(myStruct) - assertEquals("struct", myStruct.kind) - - val fields = myStruct.fields - - assertEquals(1, fields.size) - - var methods = myStruct.methods - - var myFunc = methods.firstOrNull() - assertNotNull(myFunc) - - assertLocalName("MyFunc", myFunc) - - val myField = fields.firstOrNull() - assertNotNull(myField) - - assertLocalName("MyField", myField) - assertEquals(tu.primitiveType("int"), myField.type) - - val myInterface = p.records["p.MyInterface"] - assertNotNull(myInterface) - assertEquals("interface", myInterface.kind) - - methods = myInterface.methods - - assertEquals(1, methods.size) - - myFunc = methods.first() - - assertLocalName("MyFunc", myFunc) - assertLocalName("func() string", myFunc.type) - - val newMyStruct = p.functions["NewMyStruct"] - assertNotNull(newMyStruct) - - val body = newMyStruct.body as? Block - - assertNotNull(body) - - val `return` = body.statements.first() as? ReturnStatement + val funcB = tu.functions["funcB"] + assertNotNull(funcB) + assertEquals(3, funcB.parameters.size) + assertEquals( + listOf("uint8[]", "uint8[]", "int"), + funcB.parameters.map { it.type.name.toString() } + ) - assertNotNull(`return`) + type = funcB.type + assertIs(type) + assertNotNull(type) + assertEquals(3, type.parameters.size) + assertEquals( + listOf("uint8[]", "uint8[]", "int"), + type.parameters.map { it.name.toString() } + ) - val returnValue = `return`.returnValue as? UnaryOperator + val funcC = tu.functions["funcC"] + assertNotNull(funcC) + assertEquals(1, funcC.parameters.size) + assertEquals(listOf("string"), funcC.parameters.map { it.type.name.toString() }) - assertNotNull(returnValue) + type = funcC.type + assertIs(type) + assertNotNull(type) + assertEquals(1, type.parameters.size) + assertEquals(listOf("string"), type.parameters.map { it.name.toString() }) } @Test @@ -491,15 +545,12 @@ class GoLanguageFrontendTest : BaseTest() { assertLocalName("b", b) assertEquals(tu.primitiveType("bool"), b.type) - // true, false are builtin variables, NOT literals in Golang - // we might need to parse this special case differently - val initializer = b.initializer as? Reference - - assertNotNull(initializer) - assertLocalName("true", initializer) + // Technically, "true" and "false" are builtin variables, NOT literals in Golang, + // however, we parse them as literals to have compatibility with other languages + // also enable all features, such as value resolution based on literal values. + assertLiteralValue(true, b.initializer) val `if` = body.statements[1] as? IfStatement - assertNotNull(`if`) } @@ -572,7 +623,10 @@ class GoLanguageFrontendTest : BaseTest() { } assertNotNull(result) - val tus = result.translationUnits + val app = result.components["application"] + assertNotNull(app) + + val tus = app.translationUnits val tu = tus[0] val p = tu.namespaces["p"] @@ -670,9 +724,10 @@ class GoLanguageFrontendTest : BaseTest() { it.registerLanguage() } - assertNotNull(result) - val tus = result.translationUnits + val app = result.components["application"] + assertNotNull(app) + val tus = app.translationUnits val tu0 = tus[0] assertNotNull(tu0) @@ -713,7 +768,6 @@ class GoLanguageFrontendTest : BaseTest() { analyzeAndGetFirstTU(listOf(topLevel.resolve("comment.go").toFile()), topLevel, true) { it.registerLanguage() } - assertNotNull(tu) val mainNamespace = tu.namespaces["main"] @@ -787,6 +841,188 @@ class GoLanguageFrontendTest : BaseTest() { assertNotNull(assign) val call = assertIs(assign.value) - assertLocalName("myTest", call) + assertLocalName("funcA", call) + } + + @Test + fun testTypes() { + val topLevel = Path.of("src", "test", "resources", "golang") + val tu = + analyzeAndGetFirstTU(listOf(topLevel.resolve("types.go").toFile()), topLevel, true) { + it.registerLanguage() + } + assertNotNull(tu) + + val s = tu.variables["s"] + assertNotNull(s) + assertIs(s.type) + assertLocalName("struct{Field int}", s.type) + + val i = tu.variables["i"] + assertNotNull(i) + assertIs(i.type) + assertLocalName("interface{MyMethod(int) error}", i.type) + + val m = tu.variables["m"] + assertNotNull(m) + var type = m.type + assertIs(type) + assertEquals(listOf("int", "string"), type.generics.map { it.name.toString() }) + + val a = tu.variables["a"] + assertNotNull(a) + type = a.type + assertIs(type) + assertEquals(PointerType.PointerOrigin.ARRAY, type.pointerOrigin) + assertLocalName("int", type.elementType) + + val f = tu.variables["f"] + assertNotNull(f) + assertIs(f.type) + + val g = tu.variables["g"] + assertNotNull(g) + assertLocalName("string", g.type) + + val h = tu.variables["h"] + assertNotNull(h) + + type = h.type + assertIs(type) + assertLocalName("newType", type) + + assertEquals(1, type.recordDeclaration?.methods?.size) + } + + @Test + fun testResolveStdLibImport() { + val stdLib = Path.of("src", "test", "resources", "golang-std") + val topLevel = Path.of("src", "test", "resources", "golang") + val tu = + analyzeAndGetFirstTU( + listOf( + topLevel.resolve("function.go").toFile(), + stdLib.resolve("fmt").toFile(), + ), + topLevel, + true + ) { + it.registerLanguage() + it.includePath(stdLib) + } + + assertNotNull(tu) + + val p = tu.namespaces["p"] + assertNotNull(p) + + val main = p.functions["main"] + assertNotNull(main) + + val printfCall = main.calls["fmt.Printf"] + assertNotNull(printfCall) + + val printf = printfCall.invokes.firstOrNull() + assertNotNull(printf) + assertEquals("print.go", File(printf.location?.artifactLocation?.uri?.path.toString()).name) + } + + @Test + fun testBuildTags() { + val stdLib = Path.of("src", "test", "resources", "golang-std") + val topLevel = Path.of("src", "test", "resources", "golang", "buildtags") + val result = + analyze( + listOf( + topLevel.resolve("").toFile(), + stdLib.resolve("fmt").toFile(), + ), + topLevel, + true + ) { + it.registerLanguage() + it.includePath(stdLib) + it.symbols(mapOf("GOOS" to "darwin", "GOARCH" to "arm64")) + } + + assertNotNull(result) + + val funcOS = result.functions("OS") + assertEquals(1, funcOS.size) + + val specific = result.functions["someSpecific"] + assertNotNull(specific) + } + + @Test + fun testInterfaceDeriveFrom() { + val topLevel = Path.of("src", "test", "resources", "golang") + val tu = + analyzeAndGetFirstTU(listOf(topLevel.resolve("struct.go").toFile()), topLevel, true) { + it.registerLanguage() + } + assertNotNull(tu) + + val doInterface = tu.functions["DoInterface"] + assertNotNull(doInterface) + + val call = tu.calls["DoInterface"] + assertNotNull(call) + assertInvokes(call, doInterface) + } + + @Test + fun testFuncOptions() { + val topLevel = Path.of("src", "test", "resources", "golang", "options") + val result = + analyze( + listOf( + topLevel.resolve("srv.go").toFile(), + topLevel.resolve("srv_option.go").toFile() + ), + topLevel, + true + ) { + it.registerLanguage() + } + assertNotNull(result) + + val inner = result.records["inner"] + assertNotNull(inner) + + val field = inner.fields["field"] + assertNotNull(field) + + val assign = result.assignments.firstOrNull() + assertNotNull(assign) + + val mce = assign.target + assertIs(mce) + assertRefersTo(mce, field) + } + + @Test + fun testChainedCall() { + val topLevel = Path.of("src", "test", "resources", "golang", "chained") + val tu = + analyze( + listOf( + topLevel.resolve("chained.go").toFile(), + ), + topLevel, + true + ) { + it.registerLanguage() + } + assertNotNull(tu) + + val type = tu.records["Type"] + assertNotNull(type) + + val elem = type.methods["Elem"] + assertNotNull(elem) + + val call = tu.calls["Elem"] + assertInvokes(call, elem) } } diff --git a/cpg-language-go/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/IntegrationTest.kt b/cpg-language-go/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/IntegrationTest.kt new file mode 100644 index 0000000000..a87ab122ec --- /dev/null +++ b/cpg-language-go/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/IntegrationTest.kt @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2023, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.frontends.golang + +import de.fraunhofer.aisec.cpg.TestUtils +import de.fraunhofer.aisec.cpg.TestUtils.assertInvokes +import de.fraunhofer.aisec.cpg.TranslationConfiguration +import de.fraunhofer.aisec.cpg.TranslationResult.Companion.APPLICATION_LOCAL_NAME +import de.fraunhofer.aisec.cpg.frontends.golang.Project.Companion.buildProject +import de.fraunhofer.aisec.cpg.graph.* +import kotlin.test.Test +import kotlin.test.assertNotNull +import kotlin.test.assertNull +import org.junit.jupiter.api.Tag + +@Tag("integration") +class IntegrationTest { + + @Test + fun testProject() { + val project = buildProject("src/test/resources/golang/buildtags", "darwin", "arm64") + + val app = project.components[APPLICATION_LOCAL_NAME] + assertNotNull(app) + assertNotNull(app.firstOrNull { it.endsWith("main.go") }) + assertNotNull(app.firstOrNull { it.endsWith("func_darwin.go") }) + assertNotNull(app.firstOrNull { it.endsWith("func_darwin_arm64.go") }) + assertNull(app.firstOrNull { it.endsWith("func_darwin_ios.go") }) + assertNull(app.firstOrNull { it.endsWith("func_linux_arm64.go") }) + assertNotNull(app.firstOrNull { it.endsWith("fmt/print.go") }) + + val tus = + TestUtils.analyzeWithBuilder( + TranslationConfiguration.builder() + .softwareComponents(project.components) + .symbols(project.symbols) + .includePath(project.includePaths.first().path) + .registerLanguage() + .defaultPasses() + ) + assertNotNull(tus) + + val printTU = tus.firstOrNull { it.name.endsWith("print.go") } + assertNotNull(printTU) + + val printf = printTU.functions["Printf"] + assertNotNull(printf) + + val mainTU = tus.firstOrNull { it.name.endsWith("main.go") } + assertNotNull(mainTU) + + val printfCall = mainTU.calls["fmt.Printf"] + assertNotNull(printfCall) + assertInvokes(printfCall, printf) + } +} diff --git a/cpg-language-go/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/StatementTest.kt b/cpg-language-go/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/StatementTest.kt index 1abacc2c4c..facd3fc426 100644 --- a/cpg-language-go/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/StatementTest.kt +++ b/cpg-language-go/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/StatementTest.kt @@ -26,12 +26,10 @@ package de.fraunhofer.aisec.cpg.frontends.golang import de.fraunhofer.aisec.cpg.TestUtils.analyzeAndGetFirstTU +import de.fraunhofer.aisec.cpg.assertLocalName import de.fraunhofer.aisec.cpg.graph.* import de.fraunhofer.aisec.cpg.graph.statements.* -import de.fraunhofer.aisec.cpg.graph.statements.expressions.CallExpression -import de.fraunhofer.aisec.cpg.graph.statements.expressions.Literal -import de.fraunhofer.aisec.cpg.graph.statements.expressions.Reference -import de.fraunhofer.aisec.cpg.graph.statements.expressions.UnaryOperator +import de.fraunhofer.aisec.cpg.graph.statements.expressions.* import java.nio.file.Path import kotlin.test.Test import kotlin.test.assertEquals @@ -94,10 +92,10 @@ class StatementTest { val p = tu.namespaces["p"] assertNotNull(p) - val main = p.functions["main"] - assertNotNull(main) + val `do` = p.methods["Do"] + assertNotNull(`do`) - val op = main.allChildren { it.name.localName == "defer" }.firstOrNull() + val op = `do`.allChildren { it.name.localName == "defer" }.firstOrNull() assertNotNull(op) // The EOG for the defer statement itself should be in the regular EOG path @@ -110,4 +108,43 @@ class StatementTest { // Its call expression should connect to the return statement op.input.prevEOG.all { it is ReturnStatement } } + + @Test + fun testTypeSwitch() { + val topLevel = Path.of("src", "test", "resources", "golang") + val tu = + analyzeAndGetFirstTU( + listOf(topLevel.resolve("type_assert.go").toFile()), + topLevel, + true + ) { + it.registerLanguage() + } + assertNotNull(tu) + + val main = tu.functions["main"] + assertNotNull(main) + + val body = main.body + assertIs(body) + assertNotNull(body) + + val switch = body.statements(6) + assertNotNull(switch) + + val block = switch.statement + assertIs(block) + assertNotNull(block) + + val vs = main.variables("v") + assertNotNull(vs) + assertEquals( + listOf("main.MyStruct", "main.MyStruct*", "main.MyInterface"), + vs.map { it.type.name.toString() } + ) + vs.forEach { + assertLocalName("v", it) + assertLocalName("f", it.initializer) + } + } } diff --git a/cpg-language-go/src/test/resources/golang-std/fmt/print.go b/cpg-language-go/src/test/resources/golang-std/fmt/print.go new file mode 100644 index 0000000000..b9b025641d --- /dev/null +++ b/cpg-language-go/src/test/resources/golang-std/fmt/print.go @@ -0,0 +1,6 @@ +package fmt + +func Printf(format string, a ...any) (n int, err error) { + // Not a real implementation, and we are ignoring it anyway + return 0, nil +} diff --git a/cpg-language-go/src/test/resources/golang-std/go.mod b/cpg-language-go/src/test/resources/golang-std/go.mod new file mode 100644 index 0000000000..e299934a59 --- /dev/null +++ b/cpg-language-go/src/test/resources/golang-std/go.mod @@ -0,0 +1,3 @@ +module std + +go 1.21 \ No newline at end of file diff --git a/cpg-language-go/src/test/resources/golang/buildtags/cmd/buildtags/main.go b/cpg-language-go/src/test/resources/golang/buildtags/cmd/buildtags/main.go new file mode 100644 index 0000000000..2da41a6caf --- /dev/null +++ b/cpg-language-go/src/test/resources/golang/buildtags/cmd/buildtags/main.go @@ -0,0 +1,11 @@ +package main + +import ( + "fmt" + + "mymodule.io/buildtags" +) + +func main() { + fmt.Printf("Your OS: %s", buildtags.OS()) +} diff --git a/cpg-language-go/src/test/resources/golang/buildtags/func_darwin.go b/cpg-language-go/src/test/resources/golang/buildtags/func_darwin.go new file mode 100644 index 0000000000..4fa975686f --- /dev/null +++ b/cpg-language-go/src/test/resources/golang/buildtags/func_darwin.go @@ -0,0 +1,10 @@ +// Some other header + +//go:build darwin + +// Package buildtags is awesome +package buildtags + +func OS() string { + return "darwin" +} diff --git a/cpg-language-go/src/test/resources/golang/buildtags/func_darwin_arm64.go b/cpg-language-go/src/test/resources/golang/buildtags/func_darwin_arm64.go new file mode 100644 index 0000000000..fb81f4123b --- /dev/null +++ b/cpg-language-go/src/test/resources/golang/buildtags/func_darwin_arm64.go @@ -0,0 +1,3 @@ +package buildtags + +func someSpecific() {} diff --git a/cpg-language-go/src/test/resources/golang/buildtags/func_ios.go b/cpg-language-go/src/test/resources/golang/buildtags/func_ios.go new file mode 100644 index 0000000000..96a511d32e --- /dev/null +++ b/cpg-language-go/src/test/resources/golang/buildtags/func_ios.go @@ -0,0 +1,8 @@ +// Some other header + +// Package buildtags is awesome +package buildtags + +func OS() string { + return "ios" +} diff --git a/cpg-language-go/src/test/resources/golang/buildtags/func_linux_arm64.go b/cpg-language-go/src/test/resources/golang/buildtags/func_linux_arm64.go new file mode 100644 index 0000000000..93ed03405d --- /dev/null +++ b/cpg-language-go/src/test/resources/golang/buildtags/func_linux_arm64.go @@ -0,0 +1,5 @@ +package buildtags + +func OS() string { + return "linux" +} diff --git a/cpg-language-go/src/test/resources/golang/buildtags/go.mod b/cpg-language-go/src/test/resources/golang/buildtags/go.mod new file mode 100644 index 0000000000..7f87cd2ed5 --- /dev/null +++ b/cpg-language-go/src/test/resources/golang/buildtags/go.mod @@ -0,0 +1,3 @@ +module mymodule.io/buildtags + +go 1.21.1 diff --git a/cpg-language-go/src/test/resources/golang/chained/chained.go b/cpg-language-go/src/test/resources/golang/chained/chained.go new file mode 100644 index 0000000000..fd27a945d3 --- /dev/null +++ b/cpg-language-go/src/test/resources/golang/chained/chained.go @@ -0,0 +1,16 @@ +package chained + +type Type interface { + Elem() Type +} + +func TypeOf(i any) Type { + return nil +} + +type MyStruct interface{} + +var ( + structType = TypeOf((*MyStruct)(nil)).Elem() + _ = structType +) diff --git a/cpg-language-go/src/test/resources/golang/chan.go b/cpg-language-go/src/test/resources/golang/chan.go new file mode 100644 index 0000000000..6ea215ff5c --- /dev/null +++ b/cpg-language-go/src/test/resources/golang/chan.go @@ -0,0 +1,8 @@ +package pa + +func main() { + var v int + var ch = make(chan int) + ch <- v + <-ch +} diff --git a/cpg-language-go/src/test/resources/golang/const.go b/cpg-language-go/src/test/resources/golang/const.go new file mode 100644 index 0000000000..bd87af2206 --- /dev/null +++ b/cpg-language-go/src/test/resources/golang/const.go @@ -0,0 +1,26 @@ +package p + +type custom int64 + +const ( + one custom = 1 + oneAsWell + ten = 10 + tenAsWell +) + +const ( + two = 2 + iota + three + four +) + +const ( + five, fivehundred = 5 + iota, (5 + iota) * 100 + six, sixhundred +) + +const ( + fiveAsWell = 5 + iota*100 + onehundredandfive +) diff --git a/cpg-language-go/src/test/resources/golang/declare.go b/cpg-language-go/src/test/resources/golang/declare.go index 15da35ffa9..238f27a7fa 100644 --- a/cpg-language-go/src/test/resources/golang/declare.go +++ b/cpg-language-go/src/test/resources/golang/declare.go @@ -23,24 +23,21 @@ func main() { // VariableDeclaration nodes var c, d = 3, 4 + var e, f = test() + // Short assignment using an assignment, where all variables were not // defined before. This is an AssignStmt which has DEFINE as its token. - // - // We need to split this up into several nodes. First, we translate this - // into one (implicit) DeclarationStatement with two VariableDeclaration - // nodes. Afterwards we are parsing it as a regular assignment. - e, f := 5, 6 + g, h := 5, 6 // Short assignment using an assignment, where one variable (f) was defined // before in the local scope. This is an AssignStmt which has DEFINE as its // token. From the AST we cannot differentiate this from the previous // example and we need to do a (local) variable lookup here. - // - // Finally, We need to split this up into several nodes. First, we translate - // this into one (implicit) DeclarationStatement with one - // VariableDeclaration node. Afterwards we are parsing it as a regular - // assignment. - f, g := 7, 8 + h, i := test() + + fmt.Printf("%d %d %d %d %d %d %d\n", a, b, c, d, e, f, g, h, i) +} - fmt.Printf("%d %d %d %d %d %d %d\n", a, b, c, d, e, f, g) +func test() (i int, err error) { + return 0, nil } diff --git a/cpg-language-go/src/test/resources/golang/defer.go b/cpg-language-go/src/test/resources/golang/defer.go index 4493840aab..584fa414bc 100644 --- a/cpg-language-go/src/test/resources/golang/defer.go +++ b/cpg-language-go/src/test/resources/golang/defer.go @@ -5,7 +5,9 @@ import ( "os" ) -func main() { +type MyStruct struct{} + +func (s MyStruct) Do() { i := 1 do() defer that(i) diff --git a/cpg-language-go/src/test/resources/golang/function.go b/cpg-language-go/src/test/resources/golang/function.go index e8984d0a40..e2c5ef52cc 100644 --- a/cpg-language-go/src/test/resources/golang/function.go +++ b/cpg-language-go/src/test/resources/golang/function.go @@ -6,14 +6,14 @@ func main() { var i int var err error - i, err = myTest("some string") + i, err = funcA("some string") if err == nil { fmt.Printf("%d", i) } } -func myTest(s string) (a int, err error) { +func funcA(s string) (a int, err error) { fmt.Printf("%s", s) a = 1 + 2 @@ -22,3 +22,13 @@ func myTest(s string) (a int, err error) { return } + +// funcB is a function that show-cases different ways of declaring used and unused +// parameters in Go. +func funcB(a, b []byte, _ int) (byte, byte) { + return a[0], b[0] +} + +func funcC(string) { + return +} diff --git a/cpg-language-go/src/test/resources/golang/go.mod b/cpg-language-go/src/test/resources/golang/go.mod index 12cb5307fb..07004656da 100644 --- a/cpg-language-go/src/test/resources/golang/go.mod +++ b/cpg-language-go/src/test/resources/golang/go.mod @@ -1 +1,3 @@ module mymodule + +go 1.18 diff --git a/cpg-language-go/src/test/resources/golang/importalias.go b/cpg-language-go/src/test/resources/golang/importalias.go new file mode 100644 index 0000000000..750ff62022 --- /dev/null +++ b/cpg-language-go/src/test/resources/golang/importalias.go @@ -0,0 +1,16 @@ +package p + +import ( + fmtother "fmt" +) + +type formatter struct { + field int +} + +func main() { + fmt := formatter{} + fmt.field = 1 + + fmtother.Printf("%d", 1) +} diff --git a/cpg-language-go/src/test/resources/golang/literal.go b/cpg-language-go/src/test/resources/golang/literal.go index 4b7207b8c0..dbefc5bda1 100644 --- a/cpg-language-go/src/test/resources/golang/literal.go +++ b/cpg-language-go/src/test/resources/golang/literal.go @@ -1,6 +1,19 @@ package p +import ( + "mymodule/submodule" +) + +type MyStruct struct { + Field int +} + const a = 1 + +const ( + b int = iota +) + const s = "test" const f = 1.0 const f32 float32 = 1.00 @@ -10,3 +23,23 @@ var n *int = nil var fn = func(_ int) int { return 1 } + +type structArray []MyStruct + +// o is a composite literal. It also demonstrates that we can omit the +// type specifier of an "inner" composite literal, if the outer one is an array type +var o = []MyStruct{{Field: 10}} + +// o is similar to o2, but with a pointer type +var o2 = []*MyStruct{{Field: 10}} + +// o3 is similar to o3, but with a new type that uses []MyStruct as underlying type +var o3 = structArray{{Field: 10}} + +var rr = []int{ + submodule.Zero: 1, +} + +var mapr = map[int][]byte{ + submodule.Zero: {1, 2, 3}, +} diff --git a/cpg-language-go/src/test/resources/golang/options/srv.go b/cpg-language-go/src/test/resources/golang/options/srv.go new file mode 100644 index 0000000000..8971b5e6cb --- /dev/null +++ b/cpg-language-go/src/test/resources/golang/options/srv.go @@ -0,0 +1,9 @@ +package options + +type srv struct { + inner *inner +} + +type inner struct { + field int +} diff --git a/cpg-language-go/src/test/resources/golang/options/srv_option.go b/cpg-language-go/src/test/resources/golang/options/srv_option.go new file mode 100644 index 0000000000..d0a7360853 --- /dev/null +++ b/cpg-language-go/src/test/resources/golang/options/srv_option.go @@ -0,0 +1,9 @@ +package options + +type Option func(*srv) + +func WithField(a int) Option { + return func(s *srv) { + s.inner.field = a + } +} diff --git a/cpg-language-go/src/test/resources/golang/struct.go b/cpg-language-go/src/test/resources/golang/struct.go index 7287159ce3..18e466eea8 100644 --- a/cpg-language-go/src/test/resources/golang/struct.go +++ b/cpg-language-go/src/test/resources/golang/struct.go @@ -1,9 +1,19 @@ package p -import ("fmt") +import ( + "fmt" +) + +type OtherStruct struct { +} + +type EvenAnotherStruct struct { +} type MyStruct struct { - MyField int + MyField int + OtherStruct + *EvenAnotherStruct } type MyInterface interface { @@ -11,7 +21,7 @@ type MyInterface interface { } func (s MyStruct) MyFunc() string { - fmt.Printf(s.myOtherFunc(), s.MyField) + fmt.Printf(s.myOtherFunc(), s.MyField) return "s" } @@ -21,5 +31,20 @@ func (s MyStruct) myOtherFunc() string { } func NewMyStruct() *MyStruct { - return &MyStruct{} + return &MyStruct{} +} + +var s = struct { + field int +}{ + field: 1, +} + +func DoInterface(i MyInterface) string { + return i.MyFunc() +} + +func main() { + var myStruct = NewMyStruct() + DoInterface(myStruct) } diff --git a/cpg-language-go/src/test/resources/golang/submodule/const.go b/cpg-language-go/src/test/resources/golang/submodule/const.go new file mode 100644 index 0000000000..9d6d1ade54 --- /dev/null +++ b/cpg-language-go/src/test/resources/golang/submodule/const.go @@ -0,0 +1,3 @@ +package submodule + +const Zero = 0 diff --git a/cpg-language-go/src/test/resources/golang/type_assert.go b/cpg-language-go/src/test/resources/golang/type_assert.go index 4f42ac2798..4d25a380ea 100644 --- a/cpg-language-go/src/test/resources/golang/type_assert.go +++ b/cpg-language-go/src/test/resources/golang/type_assert.go @@ -2,19 +2,37 @@ package main import "fmt" -type MyStruct struct {} +type MyStruct struct{} + type MyInterface interface { - MyFunc() + MyFunc() } + func (MyStruct) MyFunc() {} -func main () { - var f MyInterface = MyStruct{} - var s = f.(MyStruct) +func main() { + var f MyInterface = MyStruct{} + var s = f.(MyStruct) + + fmt.Printf("%+v", s) + + var _ = MyInterface(s) + var _ = interface{}(s) + var _ = any(s) + + switch v := f.(type) { + case MyStruct: + var s2 = v + fmt.Printf("%+v", s2) + case *MyStruct: + var p2 = v + fmt.Printf("%+v", p2) + default: + var v2 = v + fmt.Printf("%+v", v2) + } +} - fmt.Printf("%+v", s) +type myStruct struct{} - var _ = MyInterface(s) - var _ = interface{}(s) - var _ = any(s) -} \ No newline at end of file +var test = (*myStruct)(nil) diff --git a/cpg-language-go/src/test/resources/golang/type_constraints.go b/cpg-language-go/src/test/resources/golang/type_constraints.go index 485f29bc2e..b4c92870e7 100644 --- a/cpg-language-go/src/test/resources/golang/type_constraints.go +++ b/cpg-language-go/src/test/resources/golang/type_constraints.go @@ -1,11 +1,11 @@ package main -type MyStruct[T any] struct {} -type MyInterface interface {} +type MyStruct[T any] struct{} +type MyInterface interface{} func SomeFunc[T any, S MyInterface]() {} func main() { - _ := &MyStruct[MyInterface]{} - SomeFunc[any, MyInterface]() -} \ No newline at end of file + _ := &MyStruct[MyInterface]{} + SomeFunc[interface{}, MyInterface]() +} diff --git a/cpg-language-go/src/test/resources/golang/types.go b/cpg-language-go/src/test/resources/golang/types.go new file mode 100644 index 0000000000..cb9f5b13d3 --- /dev/null +++ b/cpg-language-go/src/test/resources/golang/types.go @@ -0,0 +1,43 @@ +package p + +// Funcy is a function type +type Funcy func() error + +// newType is a new distinct type, it has string as the "underlying" type. +type newType string + +func (n *newType) SomeFunc() {} + +var _ = newType("test") + +// alias is really a type alias +type alias = string + +// s is a variable with a struct type. +var s struct{ Field int } + +// i is a variable with an interface type. +var i interface{ MyMethod(int) error } + +// m is a variable with an map type. +var m map[int]string + +// a is a variable with an array type. +var a []int + +// f is of Funcy type +var f Funcy + +var g alias + +var h newType + +func init() { + _ = s + _ = i + _ = m + _ = a + _ = f + _ = g + _ = h +} diff --git a/cpg-neo4j/build.gradle.kts b/cpg-neo4j/build.gradle.kts index 099df86c86..1a6c47eaf6 100644 --- a/cpg-neo4j/build.gradle.kts +++ b/cpg-neo4j/build.gradle.kts @@ -44,14 +44,6 @@ publishing { } } -tasks.withType { - useJUnitPlatform { - if (!project.hasProperty("integration")) { - excludeTags("integration") - } - } -} - dependencies { // neo4j api(libs.bundles.neo4j)