Improvements to the Go language

* Added parsing of more expressions

cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TypeManager.kt

@@ -50,12 +50,11 @@ class TypeManager {
      * Generics) to the ParameterizedType to be able to resolve the Type of the fields, since
      * ParameterizedTypes are unique to the RecordDeclaration and are not merged.
      */
-    private val recordToTypeParameters =
-        Collections.synchronizedMap(mutableMapOf<RecordDeclaration, List<ParameterizedType>>())
-    private val templateToTypeParameters =
-        Collections.synchronizedMap(
-            mutableMapOf<TemplateDeclaration, MutableList<ParameterizedType>>()
-        )
+    private val recordToTypeParameters: MutableMap<RecordDeclaration, List<ParameterizedType>> =
+        ConcurrentHashMap()
+    private val templateToTypeParameters:
+        MutableMap<TemplateDeclaration, MutableList<ParameterizedType>> =
+        ConcurrentHashMap()
 
     val firstOrderTypes: MutableSet<Type> = ConcurrentHashMap.newKeySet()
     val secondOrderTypes: MutableSet<Type> = ConcurrentHashMap.newKeySet()
@@ -288,14 +287,12 @@ internal fun Type.getAncestors(depth: Int): Set<Type.Ancestor> {
     return types
 }
 
-/** Checks, if this [Type] is either derived from or equals to [superType]. */
+/**
+ * Checks, if this [Type] is either derived from or equals to [superType]. This is forwarded to the
+ * [Language] of the [Type] and can be overridden by the individual languages.
+ */
 fun Type.isDerivedFrom(superType: Type): Boolean {
-    // Retrieve all ancestor types of our type (more concretely of the root type)
-    val root = this.root
-    val superTypes = root.ancestors.map { it.type }
-
-    // Check, if super type (or its root) is in the list
-    return superType.root in superTypes
+    return this.language?.isDerivedFrom(this, superType) ?: false
 }
 
 /**

cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/Language.kt

@@ -31,6 +31,7 @@ import com.fasterxml.jackson.databind.SerializerProvider
 import com.fasterxml.jackson.databind.annotation.JsonSerialize
 import com.fasterxml.jackson.databind.ser.std.StdSerializer
 import de.fraunhofer.aisec.cpg.TranslationContext
+import de.fraunhofer.aisec.cpg.ancestors
 import de.fraunhofer.aisec.cpg.graph.Name
 import de.fraunhofer.aisec.cpg.graph.Node
 import de.fraunhofer.aisec.cpg.graph.statements.expressions.BinaryOperator
@@ -199,6 +200,15 @@ abstract class Language<T : LanguageFrontend<*, *>> : Node() {
 
         return true
     }
+
+    open fun isDerivedFrom(type: Type, superType: Type): Boolean {
+        // Retrieve all ancestor types of our type (more concretely of the root type)
+        val root = type.root
+        val superTypes = root.ancestors.map { it.type }
+
+        // Check, if super type (or its root) is in the list
+        return superType.root in superTypes
+    }
 }
 
 /**

cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/IncludeDeclaration.kt

@@ -34,6 +34,7 @@ import java.util.Objects
 import org.apache.commons.lang3.builder.ToStringBuilder
 import org.neo4j.ogm.annotation.Relationship
 
+/** This declaration represents either an include or an import, depending on the language. */
 class IncludeDeclaration : Declaration() {
     @Relationship(value = "INCLUDES", direction = Relationship.Direction.OUTGOING)
     @AST
@@ -43,6 +44,10 @@ class IncludeDeclaration : Declaration() {
     @AST
     private val problemEdges: MutableList<PropertyEdge<ProblemDeclaration>> = ArrayList()
 
+    /**
+     * This property refers to the file or directory or path. For example, in C this refers to an
+     * include header file. In Go, this refers to the package path (e.g., github.com/a/b)
+     */
     var filename: String? = null
 
     val includes: List<IncludeDeclaration> by PropertyEdgeDelegate(IncludeDeclaration::includeEdges)

cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/DeclaredReferenceExpression.kt

@@ -137,7 +137,11 @@ open class DeclaredReferenceExpression : Expression(), HasType.TypeObserver {
         if (other !is DeclaredReferenceExpression) {
             return false
         }
-        return super.equals(other) && refersTo == other.refersTo
+        return super.equals(other)
+    }
+
+    override fun hashCode(): Int {
+        return super.hashCode()
     }
 
     override fun addPrevDFG(prev: Node, properties: MutableMap<Properties, Any?>) {
@@ -150,6 +154,4 @@ open class DeclaredReferenceExpression : Expression(), HasType.TypeObserver {
             prev.registerTypeObserver(this)
         }
     }
-
-    override fun hashCode(): Int = Objects.hash(super.hashCode(), refersTo)
 }

cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/InitializerListExpression.kt

@@ -68,7 +68,14 @@ class InitializerListExpression : Expression(), ArgumentHolder, HasType.TypeObse
     }
 
     override fun replaceArgument(old: Expression, new: Expression): Boolean {
-        // Not supported, too complex
+        val idx = initializerEdges.indexOfFirst { it.end == old }
+        if (idx != -1) {
+            old.unregisterTypeObserver(this)
+            initializerEdges[idx].end = new
+            new.registerTypeObserver(this)
+            return true
+        }
+
         return false
     }
 

cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/KeyValueExpression.kt

@@ -26,6 +26,7 @@
 package de.fraunhofer.aisec.cpg.graph.statements.expressions
 
 import de.fraunhofer.aisec.cpg.graph.AST
+import de.fraunhofer.aisec.cpg.graph.ArgumentHolder
 import java.util.*
 
 /**
@@ -35,7 +36,7 @@ import java.util.*
  * Most often used in combination with an [InitializerListExpression] to represent the creation of
  * an array.
  */
-class KeyValueExpression : Expression() {
+class KeyValueExpression : Expression(), ArgumentHolder {
 
     /**
      * The key of this pair. It is usually a literal, but some languages even allow references to
@@ -46,6 +47,26 @@ class KeyValueExpression : Expression() {
     /** The value of this pair. It can be any expression */
     @AST var value: Expression? = null
 
+    override fun addArgument(expression: Expression) {
+        if (key == null) {
+            key = expression
+        } else if (value == null) {
+            value = expression
+        }
+    }
+
+    override fun replaceArgument(old: Expression, new: Expression): Boolean {
+        if (key == old) {
+            key = new
+            return true
+        } else if (value == old) {
+            value = new
+            return true
+        }
+
+        return false
+    }
+
     override fun equals(other: Any?): Boolean {
         if (this === other) return true
         if (other !is KeyValueExpression) return false

cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/MemberExpression.kt

@@ -26,6 +26,7 @@
 package de.fraunhofer.aisec.cpg.graph.statements.expressions
 
 import de.fraunhofer.aisec.cpg.graph.AST
+import de.fraunhofer.aisec.cpg.graph.ArgumentHolder
 import de.fraunhofer.aisec.cpg.graph.HasBase
 import de.fraunhofer.aisec.cpg.graph.declarations.RecordDeclaration
 import de.fraunhofer.aisec.cpg.graph.fqn
@@ -39,7 +40,7 @@ import org.apache.commons.lang3.builder.ToStringBuilder
  * use-case is access of a member function (method) as part of the [MemberCallExpression.callee]
  * property of a [MemberCallExpression].
  */
-class MemberExpression : DeclaredReferenceExpression(), HasBase {
+class MemberExpression : DeclaredReferenceExpression(), ArgumentHolder, HasBase {
     @AST
     override var base: Expression = ProblemExpression("could not parse base expression")
         set(value) {
@@ -58,6 +59,19 @@ class MemberExpression : DeclaredReferenceExpression(), HasBase {
             .toString()
     }
 
+    override fun addArgument(expression: Expression) {
+        this.base = expression
+    }
+
+    override fun replaceArgument(old: Expression, new: Expression): Boolean {
+        if (old == base) {
+            base = new
+            return true
+        }
+
+        return false
+    }
+
     override fun equals(other: Any?): Boolean {
         if (this === other) return true
         if (other !is MemberExpression) return false

cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/UnaryOperator.kt

@@ -27,13 +27,14 @@ package de.fraunhofer.aisec.cpg.graph.statements.expressions
 
 import de.fraunhofer.aisec.cpg.graph.AST
 import de.fraunhofer.aisec.cpg.graph.AccessValues
+import de.fraunhofer.aisec.cpg.graph.ArgumentHolder
 import de.fraunhofer.aisec.cpg.graph.pointer
 import de.fraunhofer.aisec.cpg.graph.types.HasType
 import de.fraunhofer.aisec.cpg.graph.types.Type
 import org.apache.commons.lang3.builder.ToStringBuilder
 
 /** A unary operator expression, involving one expression and an operator, such as `a++`. */
-class UnaryOperator : Expression(), HasType.TypeObserver {
+class UnaryOperator : Expression(), ArgumentHolder, HasType.TypeObserver {
     /** The expression on which the operation is applied. */
     @AST
     var input: Expression = ProblemExpression("could not parse input")
@@ -112,6 +113,19 @@ class UnaryOperator : Expression(), HasType.TypeObserver {
         )
     }
 
+    override fun addArgument(expression: Expression) {
+        this.input = expression
+    }
+
+    override fun replaceArgument(old: Expression, new: Expression): Boolean {
+        if (this.input == old) {
+            this.input = new
+            return true
+        }
+
+        return false
+    }
+
     override fun equals(other: Any?): Boolean {
         if (this === other) {
             return true

cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/ControlFlowSensitiveDFGPass.kt

@@ -202,11 +202,19 @@ open class ControlFlowSensitiveDFGPass(ctx: TranslationContext) : TranslationUni
             // the "normal" case won't work. We handle this case separately here...
             // This is what we write to the declaration
             val iterable = currentNode.iterable as? Expression
-
             val writtenTo =
-                when (currentNode.variable) {
-                    is DeclarationStatement ->
-                        (currentNode.variable as DeclarationStatement).singleDeclaration
+                when (val variable = currentNode.variable) {
+                    is DeclarationStatement -> {
+                        if (variable.isSingleDeclaration()) {
+                            variable.singleDeclaration
+                        } else if (variable.variables.size == 2) {
+                            // If there are two variables, we just blindly assume that the order is
+                            // (key, value), so we return the second one
+                            variable.declarations[1]
+                        } else {
+                            null
+                        }
+                    }
                     else -> currentNode.variable
                 }
 

cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/EvaluationOrderGraphPass.kt

@@ -132,6 +132,7 @@ open class EvaluationOrderGraphPass(ctx: TranslationContext) : TranslationUnitPa
         map[DoStatement::class.java] = { handleDoStatement(it as DoStatement) }
         map[ForStatement::class.java] = { handleForStatement(it as ForStatement) }
         map[ForEachStatement::class.java] = { handleForEachStatement(it as ForEachStatement) }
+        map[TypeExpression::class.java] = { handleTypeExpression(it as TypeExpression) }
         map[TryStatement::class.java] = { handleTryStatement(it as TryStatement) }
         map[ContinueStatement::class.java] = { handleContinueStatement(it as ContinueStatement) }
         map[DeleteExpression::class.java] = { handleDeleteExpression(it as DeleteExpression) }
@@ -594,6 +595,10 @@ open class EvaluationOrderGraphPass(ctx: TranslationContext) : TranslationUnitPa
         pushToEOG(node)
     }
 
+    protected fun handleTypeExpression(node: TypeExpression) {
+        pushToEOG(node)
+    }
+
     protected fun handleTryStatement(node: TryStatement) {
         scopeManager.enterScope(node)
         val tryScope = scopeManager.currentScope as TryScope?

cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/TypeResolver.kt

@@ -29,7 +29,8 @@ import de.fraunhofer.aisec.cpg.TranslationContext
 import de.fraunhofer.aisec.cpg.graph.Component
 import de.fraunhofer.aisec.cpg.graph.Node
 import de.fraunhofer.aisec.cpg.graph.declarations.RecordDeclaration
-import de.fraunhofer.aisec.cpg.graph.types.*
+import de.fraunhofer.aisec.cpg.graph.types.ObjectType
+import de.fraunhofer.aisec.cpg.graph.types.Type
 import de.fraunhofer.aisec.cpg.processing.IVisitor
 import de.fraunhofer.aisec.cpg.processing.strategy.Strategy
 

cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/VariableUsageResolver.kt

@@ -198,6 +198,7 @@ open class VariableUsageResolver(ctx: TranslationContext) : SymbolResolverPass(c
                 log,
                 "Did not find a declaration for ${current.name}"
             )
+            ctx.unresolvedDeclarations++
         }
     }
 
@@ -308,6 +309,14 @@ open class VariableUsageResolver(ctx: TranslationContext) : SymbolResolverPass(c
     }
 
     protected fun resolveBase(reference: DeclaredReferenceExpression): Declaration? {
+        // We need to check, whether we first need to resolve our own base
+        if (reference is MemberExpression) {
+            val base = reference.base
+            if (base is DeclaredReferenceExpression && base.refersTo == null) {
+                base.refersTo = resolveBase(base)
+            }
+        }
+
         val declaration = scopeManager.resolveReference(reference)
         if (declaration != null) {
             return declaration

cpg-core/src/testFixtures/kotlin/de/fraunhofer/aisec/cpg/TestUtils.kt

@@ -26,7 +26,7 @@
 package de.fraunhofer.aisec.cpg
 
 import de.fraunhofer.aisec.cpg.frontends.CompilationDatabase
-import de.fraunhofer.aisec.cpg.graph.Node
+import de.fraunhofer.aisec.cpg.graph.*
 import de.fraunhofer.aisec.cpg.graph.declarations.Declaration
 import de.fraunhofer.aisec.cpg.graph.declarations.FunctionDeclaration
 import de.fraunhofer.aisec.cpg.graph.declarations.TranslationUnitDeclaration
@@ -155,7 +155,11 @@ object TestUtils {
     ): List<TranslationUnitDeclaration> {
         val config = builder.build()
         val analyzer = TranslationManager.builder().config(config).build()
-        return analyzer.analyze().get().translationUnits
+        val result = analyzer.analyze().get()
+
+        println(result.finalCtx.unresolvedDeclarations)
+
+        return result.components["application"]?.translationUnits ?: listOf()
     }
 
     @JvmOverloads

cpg-language-go/build.gradle.kts

@@ -41,6 +41,7 @@ publishing {
 
 dependencies {
     implementation("net.java.dev.jna:jna:5.13.0")
+    testImplementation(project(":cpg-analysis"))
 }
 
 if (!project.hasProperty("skipGoBuild")) {