Add generic parameters for LanguageFrontend (#1236)

This PR adds generic parameters representing AST node types and type node types of the original parser.

.github/workflows/build.yml

@@ -132,7 +132,7 @@ jobs:
       - name: Prepare test and coverage reports
         if: ${{ always() }}
         run: |
-          zip reports.zip **/build/reports/**
+          zip reports.zip **/build/reports/**/** || true
       - name: Archive test and coverage reports
         if: ${{ always() }}
         uses: actions/upload-artifact@v3

cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/ScopeManager.kt

@@ -75,7 +75,7 @@ class ScopeManager : ScopeProvider {
      * The language frontend tied to the scope manager. Can be used to implement language specific
      * scope resolution or lookup.
      */
-    var lang: LanguageFrontend? = null
+    var lang: LanguageFrontend<*, *>? = null
 
     /** True, if the scope manager is currently in a [BlockScope]. */
     val isInBlock: Boolean

cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationConfiguration.kt

@@ -99,7 +99,7 @@ private constructor(
      */
     val replacedPasses:
         Map<Pair<KClass<out Pass<*>>, KClass<out Language<*>>>, KClass<out Pass<*>>>,
-    languages: List<Language<out LanguageFrontend>>,
+    languages: List<Language<*>>,
     codeInNodes: Boolean,
     processAnnotations: Boolean,
     disableCleanup: Boolean,
@@ -112,7 +112,7 @@ private constructor(
     addIncludesToGraph: Boolean
 ) {
     /** This list contains all languages which we want to translate. */
-    val languages: List<Language<out LanguageFrontend>>
+    val languages: List<Language<*>>
 
     /**
      * Switch off cleaning up TypeManager memory after analysis.
@@ -215,7 +215,7 @@ private constructor(
      */
     class Builder {
         private var softwareComponents: MutableMap<String, List<File>> = HashMap()
-        private val languages = mutableListOf<Language<out LanguageFrontend>>()
+        private val languages = mutableListOf<Language<*>>()
         private var topLevel: File? = null
         private var debugParser = false
         private var failOnError = false
@@ -405,7 +405,7 @@ private constructor(
         }
 
         /** Registers an additional [Language]. */
-        fun registerLanguage(language: Language<out LanguageFrontend>): Builder {
+        fun registerLanguage(language: Language<*>): Builder {
             languages.add(language)
             log.info(
                 "Registered language frontend '${language::class.simpleName}' for following file types: ${language.fileExtensions}"
@@ -414,7 +414,7 @@ private constructor(
         }
 
         /** Registers an additional [Language]. */
-        inline fun <reified T : Language<out LanguageFrontend>> registerLanguage(): Builder {
+        inline fun <reified T : Language<*>> registerLanguage(): Builder {
             T::class.primaryConstructor?.call()?.let { registerLanguage(it) }
             return this
         }
@@ -443,8 +443,8 @@ private constructor(
         }
 
         /** Unregisters a registered [de.fraunhofer.aisec.cpg.frontends.Language]. */
-        fun unregisterLanguage(language: Class<out Language<out LanguageFrontend>?>): Builder {
-            languages.removeIf { obj: Language<out LanguageFrontend>? -> language.isInstance(obj) }
+        fun unregisterLanguage(language: Class<out Language<*>?>): Builder {
+            languages.removeIf { obj: Language<*>? -> language.isInstance(obj) }
             return this
         }
 
@@ -490,7 +490,7 @@ private constructor(
                 return
             }
 
-            for (frontend in languages.map(Language<out LanguageFrontend>::frontend)) {
+            for (frontend in languages.map(Language<*>::frontend)) {
                 val extraPasses = frontend.findAnnotations<RegisterExtraPass>()
                 if (extraPasses.isNotEmpty()) {
                     for (p in extraPasses) {
@@ -505,7 +505,7 @@ private constructor(
         }
 
         private fun registerReplacedPasses() {
-            for (frontend in languages.map(Language<out LanguageFrontend>::frontend)) {
+            for (frontend in languages.map(Language<*>::frontend)) {
                 val replacedPasses = frontend.findAnnotations<ReplacePass>()
                 if (replacedPasses.isNotEmpty()) {
                     for (p in replacedPasses) {

cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationManager.kt

@@ -75,7 +75,7 @@ private constructor(
     }
 
     private fun analyzeNonAsync(): TranslationResult {
-        var executedFrontends = setOf<LanguageFrontend>()
+        var executedFrontends = setOf<LanguageFrontend<*, *>>()
 
         // Build a new global translation context
         val ctx = TranslationContext(config, ScopeManager(), TypeManager())
@@ -135,8 +135,8 @@ private constructor(
     private fun runFrontends(
         ctx: TranslationContext,
         result: TranslationResult
-    ): Set<LanguageFrontend> {
-        val usedFrontends = mutableSetOf<LanguageFrontend>()
+    ): Set<LanguageFrontend<*, *>> {
+        val usedFrontends = mutableSetOf<LanguageFrontend<*, *>>()
         for (sc in ctx.config.softwareComponents.keys) {
             val component = Component()
             component.name = Name(sc)
@@ -252,14 +252,14 @@ private constructor(
         result: TranslationResult,
         globalCtx: TranslationContext,
         sourceLocations: Collection<File>
-    ): Set<LanguageFrontend> {
-        val usedFrontends = mutableSetOf<LanguageFrontend>()
+    ): Set<LanguageFrontend<*, *>> {
+        val usedFrontends = mutableSetOf<LanguageFrontend<*, *>>()
 
         log.info("Parallel parsing started")
-        val futures = mutableListOf<CompletableFuture<Optional<LanguageFrontend>>>()
+        val futures = mutableListOf<CompletableFuture<LanguageFrontend<*, *>?>>()
         val parallelContexts = mutableListOf<TranslationContext>()
 
-        val futureToFile: MutableMap<CompletableFuture<Optional<LanguageFrontend>>, File> =
+        val futureToFile: MutableMap<CompletableFuture<LanguageFrontend<*, *>?>, File> =
             IdentityHashMap()
 
         for (sourceLocation in sourceLocations) {
@@ -284,7 +284,8 @@ private constructor(
 
         for (future in futures) {
             try {
-                future.get().ifPresent { f: LanguageFrontend ->
+                val f = future.get()
+                if (f != null) {
                     handleCompletion(result, usedFrontends, futureToFile[future], f)
                 }
             } catch (e: InterruptedException) {
@@ -310,13 +311,14 @@ private constructor(
         result: TranslationResult,
         ctx: TranslationContext,
         sourceLocations: Collection<File>
-    ): Set<LanguageFrontend> {
-        val usedFrontends = mutableSetOf<LanguageFrontend>()
+    ): Set<LanguageFrontend<*, *>> {
+        val usedFrontends = mutableSetOf<LanguageFrontend<*, *>>()
 
         for (sourceLocation in sourceLocations) {
             log.info("Parsing {}", sourceLocation.absolutePath)
 
-            parse(component, ctx, sourceLocation).ifPresent { f: LanguageFrontend ->
+            var f = parse(component, ctx, sourceLocation)
+            if (f != null) {
                 handleCompletion(result, usedFrontends, sourceLocation, f)
             }
         }
@@ -326,9 +328,9 @@ private constructor(
 
     private fun handleCompletion(
         result: TranslationResult,
-        usedFrontends: MutableSet<LanguageFrontend>,
+        usedFrontends: MutableSet<LanguageFrontend<*, *>>,
         sourceLocation: File?,
-        f: LanguageFrontend
+        f: LanguageFrontend<*, *>
     ) {
         usedFrontends.add(f)
 
@@ -345,8 +347,8 @@ private constructor(
         component: Component,
         ctx: TranslationContext,
         sourceLocation: File,
-    ): Optional<LanguageFrontend> {
-        var frontend: LanguageFrontend? = null
+    ): LanguageFrontend<*, *>? {
+        var frontend: LanguageFrontend<*, *>? = null
         try {
             frontend = getFrontend(sourceLocation, ctx)
 
@@ -358,7 +360,7 @@ private constructor(
                         "Found no parser frontend for ${sourceLocation.name}"
                     )
                 }
-                return Optional.empty()
+                return null
             }
             component.translationUnits.add(frontend.parse(sourceLocation))
         } catch (ex: TranslationException) {
@@ -367,10 +369,10 @@ private constructor(
                 throw ex
             }
         }
-        return Optional.ofNullable(frontend)
+        return frontend
     }
 
-    private fun getFrontend(file: File, ctx: TranslationContext): LanguageFrontend? {
+    private fun getFrontend(file: File, ctx: TranslationContext): LanguageFrontend<*, *>? {
         val language = file.language
 
         return if (language != null) {

cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/Handler.kt

@@ -25,14 +25,11 @@
  */
 package de.fraunhofer.aisec.cpg.frontends
 
-import de.fraunhofer.aisec.cpg.TranslationContext
 import de.fraunhofer.aisec.cpg.graph.*
-import de.fraunhofer.aisec.cpg.graph.scopes.Scope
 import de.fraunhofer.aisec.cpg.helpers.Util.errorWithFileLocation
 import java.lang.reflect.ParameterizedType
 import java.lang.reflect.Type
 import java.util.function.Supplier
-import org.eclipse.cdt.internal.core.dom.parser.ASTNode
 import org.slf4j.Logger
 import org.slf4j.LoggerFactory
 
@@ -47,42 +44,36 @@ import org.slf4j.LoggerFactory
  * @param <T> the raw ast node specific to the parser
  * @param <L> the language frontend </L></T></S>
  */
-abstract class Handler<S : Node, T, L : LanguageFrontend>(
-    protected val configConstructor: Supplier<S>,
+abstract class Handler<ResultNode : Node, HandlerNode, L : LanguageFrontend<HandlerNode, *>>(
+    protected val configConstructor: Supplier<ResultNode>,
     /** Returns the frontend which used this handler. */
-    var frontend: L
-) : LanguageProvider, CodeAndLocationProvider, ScopeProvider, NamespaceProvider, ContextProvider {
-    protected val map = HashMap<Class<out T>, HandlerInterface<S, T>>()
+    val frontend: L
+) :
+    LanguageProvider by frontend,
+    CodeAndLocationProvider<HandlerNode> by frontend,
+    ScopeProvider by frontend,
+    NamespaceProvider by frontend,
+    ContextProvider by frontend {
+    protected val map = HashMap<Class<out HandlerNode>, HandlerInterface<ResultNode, HandlerNode>>()
     private val typeOfT: Class<*>?
 
     /**
-     * Searches for a handler matching the most specific superclass of [T]. The created map should
-     * thus contain a handler for every semantically different AST node and can reuse handler code
-     * as long as the handled AST nodes have a common ancestor.
+     * Searches for a handler matching the most specific superclass of [HandlerNode]. The created
+     * map should thus contain a handler for every semantically different AST node and can reuse
+     * handler code as long as the handled AST nodes have a common ancestor.
      *
      * @param ctx The AST node, whose handler is matched with respect to the AST node class.
      * @return most specific handler.
      */
-    open fun handle(ctx: T): S? {
-        var ret: S?
+    open fun handle(ctx: HandlerNode): ResultNode? {
+        var ret: ResultNode?
         if (ctx == null) {
             log.error(
                 "ctx is NULL. This can happen when ast children are optional in ${this.javaClass}. Called by ${Thread.currentThread().stackTrace[2]}"
             )
             return null
         }
 
-        // If we do not want to load includes into the CPG and the current fileLocation was included
-        if (!frontend.config.loadIncludes && ctx is ASTNode) {
-            val astNode = ctx as ASTNode
-            if (
-                astNode.fileLocation != null &&
-                    astNode.fileLocation.contextInclusionStatement != null
-            ) {
-                log.debug("Skip parsing include file ${astNode.containingFilename}")
-                return null
-            }
-        }
         var toHandle: Class<*> = ctx.javaClass
         var handler = map[toHandle]
         while (handler == null) {
@@ -92,7 +83,7 @@ abstract class Handler<S : Node, T, L : LanguageFrontend>(
                 handler != null && // always ok to handle as generic literal expr
                 !ctx.javaClass.simpleName.contains("LiteralExpr")
             ) {
-                errorWithFileLocation<T>(
+                errorWithFileLocation(
                     frontend,
                     ctx,
                     log,
@@ -110,13 +101,13 @@ abstract class Handler<S : Node, T, L : LanguageFrontend>(
                 // we will
                 // set the location here.
                 if (s.location == null) {
-                    frontend.setCodeAndLocation<S, T>(s, ctx)
+                    frontend.setCodeAndLocation(s, ctx)
                 }
-                frontend.setComment<S, T>(s, ctx)
+                frontend.setComment(s, ctx)
             }
             ret = s
         } else {
-            errorWithFileLocation<T>(
+            errorWithFileLocation(
                 frontend,
                 ctx,
                 log,
@@ -132,7 +123,7 @@ abstract class Handler<S : Node, T, L : LanguageFrontend>(
 
         // In case the node is empty, we report a problem
         if (ret == null) {
-            errorWithFileLocation<T>(
+            errorWithFileLocation(
                 frontend,
                 ctx,
                 log,
@@ -167,23 +158,6 @@ abstract class Handler<S : Node, T, L : LanguageFrontend>(
         return null
     }
 
-    /** Returns the language which this handler is parsing. */
-    override val language: Language<L>
-        get() = frontend.language as Language<L>
-
-    override fun <N, S> setCodeAndLocation(cpgNode: N, astNode: S?) {
-        frontend.setCodeAndLocation<N, S>(cpgNode, astNode)
-    }
-
-    override val scope: Scope?
-        get() = frontend.scope
-
-    override val namespace: Name?
-        get() = frontend.namespace
-
-    override val ctx: TranslationContext
-        get() = frontend.ctx
-
     companion object {
         @JvmStatic protected val log: Logger = LoggerFactory.getLogger(Handler::class.java)
     }

cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/Language.kt

@@ -48,7 +48,7 @@ import kotlin.reflect.full.primaryConstructor
  * persisted in the final graph (database) and each node links to its corresponding language using
  * the [Node.language] property.
  */
-abstract class Language<T : LanguageFrontend> : Node() {
+abstract class Language<T : LanguageFrontend<*, *>> : Node() {
     /** The file extensions without the dot */
     abstract val fileExtensions: List<String>