Improvements of Go language frontend (#1276)

.github/workflows/build.yml

@@ -22,7 +22,7 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@v4
         with:
-          go-version: "1.20"
+          go-version: "1.21"
       - name: Build
         run: |
           cd cpg-language-go/src/main/golang
@@ -65,7 +65,7 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@v4
         with:
-          go-version: 1.18
+          go-version: 1.21
       - name: Cache SonarCloud packages
         uses: actions/cache@v3
         with:
@@ -126,7 +126,7 @@ jobs:
       - name: Prepare test and coverage reports
         if: ${{ always() }}
         run: |
-          zip reports.zip **/build/reports/**/** || true
+          zip -r reports.zip **/build/reports/**/** || true
       - name: Archive test and coverage reports
         if: ${{ always() }}
         uses: actions/upload-artifact@v3

buildSrc/src/main/kotlin/cpg.common-conventions.gradle.kts

@@ -112,7 +112,11 @@ tasks.withType<KotlinCompile> {
 // common testing configuration
 //
 tasks.test {
-    useJUnitPlatform()
+    useJUnitPlatform() {
+        if (!project.hasProperty("integration")) {
+            excludeTags("integration")
+        }
+    }
     maxHeapSize = "4048m"
 }
 

cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/ScopeManager.kt

@@ -25,6 +25,7 @@
  */
 package de.fraunhofer.aisec.cpg
 
+import de.fraunhofer.aisec.cpg.frontends.HasFirstClassFunctions
 import de.fraunhofer.aisec.cpg.frontends.LanguageFrontend
 import de.fraunhofer.aisec.cpg.graph.*
 import de.fraunhofer.aisec.cpg.graph.declarations.*
@@ -37,6 +38,7 @@ import de.fraunhofer.aisec.cpg.graph.types.FunctionPointerType
 import de.fraunhofer.aisec.cpg.graph.types.IncompleteType
 import de.fraunhofer.aisec.cpg.graph.types.Type
 import de.fraunhofer.aisec.cpg.helpers.Util
+import de.fraunhofer.aisec.cpg.sarif.PhysicalLocation
 import java.util.*
 import java.util.function.Predicate
 import org.slf4j.LoggerFactory
@@ -69,6 +71,21 @@ class ScopeManager : ScopeProvider {
     var currentScope: Scope? = null
         private set
 
+    /** Represents an alias with the name [to] for the particular name [from]. */
+    data class Alias(var from: Name, var to: Name)
+
+    /**
+     * In some languages, we can define aliases for names. An example is renaming package imports in
+     * Go, e.g., to avoid name conflicts.
+     *
+     * In reality, they can probably be defined at different scopes for other languages, but for now
+     * we only allow it for the current file.
+     *
+     * This can potentially be used to replace [addTypedef] at some point, which still relies on the
+     * existence of a [LanguageFrontend].
+     */
+    private val aliases = mutableMapOf<PhysicalLocation.ArtifactLocation, MutableSet<Alias>>()
+
     /**
      * The language frontend tied to the scope manager. Can be used to implement language specific
      * scope resolution or lookup.
@@ -597,28 +614,45 @@ class ScopeManager : ScopeProvider {
      * TODO: We should merge this function with [.resolveFunction]
      */
     @JvmOverloads
-    fun resolveReference(ref: Reference, scope: Scope? = currentScope): ValueDeclaration? {
+    fun resolveReference(ref: Reference, startScope: Scope? = currentScope): ValueDeclaration? {
+        // Unfortunately, we still have an issue about duplicate declarations because header files
+        // are included multiple times, so we need to exclude the C++ frontend (for now).
+        val language = ref.language
+        val (scope, name) =
+            if (
+                language?.name?.localName != "CLanguage" &&
+                    (language?.name?.localName != "CPPLanguage")
+            ) {
+                // For all other languages, we can extract the scope information out of the name and
+                // start our search at the dedicated scope.
+                extractScope(ref, startScope)
+            } else {
+                Pair(scope, ref.name)
+            }
+
+        // Try to resolve value declarations according to our criteria
         return resolve<ValueDeclaration>(scope) {
-                if (
-                    it.name.lastPartsMatch(ref.name)
-                ) { // TODO: This place is likely to make things fail
-                    var helper = ref.resolutionHelper
-                    // If the reference seems to point to a function the entire signature is checked
-                    // for equality
-                    if (helper?.type is FunctionPointerType && it is FunctionDeclaration) {
-                        val fptrType = helper.type as FunctionPointerType
-                        // TODO(oxisto): This is the third place where function pointers are
-                        //   resolved. WHY?
-                        // TODO(oxisto): Support multiple return values
-                        val returnType = it.returnTypes.firstOrNull() ?: IncompleteType()
-                        if (
+                if (it.name.lastPartsMatch(name)) {
+                    val helper = ref.resolutionHelper
+                    return@resolve when {
+                        // If the reference seems to point to a function (using a function pointer)
+                        // the entire signature is checked for equality
+                        helper?.type is FunctionPointerType && it is FunctionDeclaration -> {
+                            val fptrType = helper.type as FunctionPointerType
+                            // TODO(oxisto): Support multiple return values
+                            val returnType = it.returnTypes.firstOrNull() ?: IncompleteType()
                             returnType == fptrType.returnType &&
                                 it.hasSignature(fptrType.parameters)
-                        ) {
-                            return@resolve true
                         }
-                    } else {
-                        return@resolve it !is FunctionDeclaration
+                        // If our language has first-class functions, we can safely return them as a
+                        // reference
+                        ref.language is HasFirstClassFunctions -> {
+                            true
+                        }
+                        // Otherwise, we are not looking for functions here
+                        else -> {
+                            it !is FunctionDeclaration
+                        }
                     }
                 }
 
@@ -636,23 +670,42 @@ class ScopeManager : ScopeProvider {
     @JvmOverloads
     fun resolveFunction(
         call: CallExpression,
-        scope: Scope? = currentScope
+        startScope: Scope? = currentScope
     ): List<FunctionDeclaration> {
-        val s = extractScope(call, scope)
+        val (scope, name) = extractScope(call, startScope)
+
+        val func =
+            resolve<FunctionDeclaration>(scope) {
+                it.name.lastPartsMatch(name) && it.hasSignature(call.signature)
+            }
 
-        return resolve(s) { it.name.lastPartsMatch(call.name) && it.hasSignature(call.signature) }
+        return func
     }
 
-    fun extractScope(node: Node, scope: Scope? = currentScope): Scope? {
+    /**
+     * This function extracts a possible scope out of a [Name], e.g. if the name is fully qualified.
+     * This also resolves possible name aliases (e.g. because of imports). It returns a pair of a
+     * scope (if found) as well as the name, which is possibly adjusted for the aliases.
+     */
+    fun extractScope(node: Node, scope: Scope? = currentScope): Pair<Scope?, Name> {
+        var name: Name = node.name
         var s = scope
 
         // First, we need to check, whether we have some kind of scoping.
         if (node.name.parent != null) {
             // extract the scope name, it is usually a name space, but could probably be something
             // else as well in other languages
-            val scopeName = node.name.parent
-
-            // TODO: proper scope selection
+            var scopeName = node.name.parent
+
+            // We need to check, whether we have an alias for the scope name in this file
+            val list = aliases[node.location?.artifactLocation]
+            val alias = list?.firstOrNull { it.to == scopeName }?.from
+            if (alias != null) {
+                scopeName = alias
+                // Reconstruct the original name with the alias, so we can resolve declarations with
+                // the namespace
+                name = Name(name.localName, alias)
+            }
 
             // this is a scoped call. we need to explicitly jump to that particular scope
             val scopes = filterScopes { (it is NameScope && it.name == scopeName) }
@@ -669,7 +722,7 @@ class ScopeManager : ScopeProvider {
                 }
         }
 
-        return s
+        return Pair(s, name)
     }
 
     /**
@@ -788,6 +841,12 @@ class ScopeManager : ScopeProvider {
             .firstOrNull()
     }
 
+    fun addAlias(file: PhysicalLocation.ArtifactLocation, from: Name, to: Name) {
+        val list = aliases.computeIfAbsent(file) { mutableSetOf() }
+
+        list += Alias(from, to)
+    }
+
     /** Returns the current scope for the [ScopeProvider] interface. */
     override val scope: Scope?
         get() = currentScope

cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TypeManager.kt

@@ -50,12 +50,11 @@ class TypeManager {
      * Generics) to the ParameterizedType to be able to resolve the Type of the fields, since
      * ParameterizedTypes are unique to the RecordDeclaration and are not merged.
      */
-    private val recordToTypeParameters =
-        Collections.synchronizedMap(mutableMapOf<RecordDeclaration, List<ParameterizedType>>())
-    private val templateToTypeParameters =
-        Collections.synchronizedMap(
-            mutableMapOf<TemplateDeclaration, MutableList<ParameterizedType>>()
-        )
+    private val recordToTypeParameters: MutableMap<RecordDeclaration, List<ParameterizedType>> =
+        ConcurrentHashMap()
+    private val templateToTypeParameters:
+        MutableMap<TemplateDeclaration, MutableList<ParameterizedType>> =
+        ConcurrentHashMap()
 
     val firstOrderTypes: MutableSet<Type> = ConcurrentHashMap.newKeySet()
     val secondOrderTypes: MutableSet<Type> = ConcurrentHashMap.newKeySet()
@@ -288,14 +287,12 @@ internal fun Type.getAncestors(depth: Int): Set<Type.Ancestor> {
     return types
 }
 
-/** Checks, if this [Type] is either derived from or equals to [superType]. */
+/**
+ * Checks, if this [Type] is either derived from or equals to [superType]. This is forwarded to the
+ * [Language] of the [Type] and can be overridden by the individual languages.
+ */
 fun Type.isDerivedFrom(superType: Type): Boolean {
-    // Retrieve all ancestor types of our type (more concretely of the root type)
-    val root = this.root
-    val superTypes = root.ancestors.map { it.type }
-
-    // Check, if super type (or its root) is in the list
-    return superType.root in superTypes
+    return this.language?.isDerivedFrom(this, superType) ?: false
 }
 
 /**

cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/Language.kt

@@ -31,6 +31,7 @@ import com.fasterxml.jackson.databind.SerializerProvider
 import com.fasterxml.jackson.databind.annotation.JsonSerialize
 import com.fasterxml.jackson.databind.ser.std.StdSerializer
 import de.fraunhofer.aisec.cpg.TranslationContext
+import de.fraunhofer.aisec.cpg.ancestors
 import de.fraunhofer.aisec.cpg.graph.Name
 import de.fraunhofer.aisec.cpg.graph.Node
 import de.fraunhofer.aisec.cpg.graph.statements.expressions.BinaryOperator
@@ -199,6 +200,15 @@ abstract class Language<T : LanguageFrontend<*, *>> : Node() {
 
         return true
     }
+
+    open fun isDerivedFrom(type: Type, superType: Type): Boolean {
+        // Retrieve all ancestor types of our type (more concretely of the root type)
+        val root = type.root
+        val superTypes = root.ancestors.map { it.type }
+
+        // Check, if super type (or its root) is in the list
+        return superType.root in superTypes
+    }
 }
 
 /**

cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/LanguageTraits.kt

@@ -44,16 +44,16 @@ import java.util.regex.Pattern
  *
  * Currently, this interface has no methods. However, in the future, this could be used to execute
  * language/frontend-specific code for the particular trait. This could help to fine-tune the
- * [de.fraunhofer.aisec.cpg.passes.CallResolver] for specific languages.
+ * [CallResolver] for specific languages.
  */
 interface LanguageTrait
 
 /** A language trait, that specifies that this language has support for templates or generics. */
 interface HasGenerics : LanguageTrait {
-    /** The char starting the template specific code (e.g. '<') */
+    /** The char starting the template specific code (e.g. `<`) */
     val startCharacter: Char
 
-    /** The char ending the template specific code (e.g. '>') */
+    /** The char ending the template specific code (e.g. `>`) */
     val endCharacter: Char
 }
 
@@ -223,3 +223,18 @@ interface HasShortCircuitOperators : LanguageTrait {
     val operatorCodes: Set<String>
         get() = conjunctiveOperators.union(disjunctiveOperators)
 }
+
+/**
+ * A language trait, that specifies that this language treats functions "first-class citizens",
+ * meaning they can be assigned to variables and passed as arguments to other functions.
+ */
+interface HasFirstClassFunctions
+
+/**
+ * A language trait, that specifies that this language has an "anonymous" identifier, used for
+ * unused parameters or suppressed assignments.
+ */
+interface HasAnonymousIdentifier {
+    val anonymousIdentifier: String
+        get() = "_"
+}

cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/FunctionDeclaration.kt

@@ -117,13 +117,13 @@ open class FunctionDeclaration : ValueDeclaration(), DeclarationHolder {
                 .stream()
                 .sorted(Comparator.comparingInt(ParameterDeclaration::argumentIndex))
                 .collect(Collectors.toList())
-        return if (targetSignature.size < signature.size) {
+        return if (signature.all { !it.isVariadic } && targetSignature.size < signature.size) {
             false
         } else {
             // signature is a collection of positional arguments, so the order must be preserved
             for (i in signature.indices) {
                 val declared = signature[i]
-                if (declared.isVariadic && targetSignature.size >= signature.size) {
+                if (declared.isVariadic) {
                     // Everything that follows is collected by this param, so the signature is
                     // fulfilled no matter what comes now (potential FIXME: in Java, we could have
                     // overloading with different vararg types, in C++ we can't, as vararg types are

cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/IncludeDeclaration.kt

@@ -34,6 +34,7 @@ import java.util.Objects
 import org.apache.commons.lang3.builder.ToStringBuilder
 import org.neo4j.ogm.annotation.Relationship
 
+/** This declaration represents either an include or an import, depending on the language. */
 class IncludeDeclaration : Declaration() {
     @Relationship(value = "INCLUDES", direction = Relationship.Direction.OUTGOING)
     @AST
@@ -43,6 +44,10 @@ class IncludeDeclaration : Declaration() {
     @AST
     private val problemEdges: MutableList<PropertyEdge<ProblemDeclaration>> = ArrayList()
 
+    /**
+     * This property refers to the file or directory or path. For example, in C this refers to an
+     * include header file. In Go, this refers to the package path (e.g., github.com/a/b)
+     */
     var filename: String? = null
 
     val includes: List<IncludeDeclaration> by PropertyEdgeDelegate(IncludeDeclaration::includeEdges)

cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/InitializerListExpression.kt

@@ -69,7 +69,14 @@ class InitializerListExpression : Expression(), ArgumentHolder, HasType.TypeObse
     }
 
     override fun replaceArgument(old: Expression, new: Expression): Boolean {
-        // Not supported, too complex
+        val idx = initializerEdges.indexOfFirst { it.end == old }
+        if (idx != -1) {
+            old.unregisterTypeObserver(this)
+            initializerEdges[idx].end = new
+            new.registerTypeObserver(this)
+            return true
+        }
+
         return false
     }