Improve the Pass system

cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/MermaidHelper.kt

@@ -0,0 +1,98 @@
+/*
+ * Copyright (c) 2024, Fraunhofer AISEC. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ *                    $$$$$$\  $$$$$$$\   $$$$$$\
+ *                   $$  __$$\ $$  __$$\ $$  __$$\
+ *                   $$ /  \__|$$ |  $$ |$$ /  \__|
+ *                   $$ |      $$$$$$$  |$$ |$$$$\
+ *                   $$ |      $$  ____/ $$ |\_$$ |
+ *                   $$ |  $$\ $$ |      $$ |  $$ |
+ *                   \$$$$$   |$$ |      \$$$$$   |
+ *                    \______/ \__|       \______/
+ *
+ */
+package de.fraunhofer.aisec.cpg
+
+import de.fraunhofer.aisec.cpg.graph.Node
+import de.fraunhofer.aisec.cpg.passes.Pass
+import de.fraunhofer.aisec.cpg.passes.hardDependencies
+import de.fraunhofer.aisec.cpg.passes.hardExecuteBefore
+import de.fraunhofer.aisec.cpg.passes.isFirstPass
+import de.fraunhofer.aisec.cpg.passes.isLastPass
+import de.fraunhofer.aisec.cpg.passes.softDependencies
+import de.fraunhofer.aisec.cpg.passes.softExecuteBefore
+import kotlin.reflect.KClass
+
+private const val UNKNOWN_PASS = "UnknownPass"
+private const val FIRST_PASSES_SUBGRAPH_IDENTIFIER = "FirstPassesSubgraph"
+private const val NORMAL_PASSES_SUBGRAPH_IDENTIFIER = "NormalPassesSubgraph"
+private const val LAST_PASSES_SUBGRAPH_IDENTIFIER = "LastPassesSubgraph"
+private const val FIRST_PASS_IDENTIFIER = "FirstPass"
+private const val LAST_PASS_IDENTIFIER = "LastPass"
+
+/** Helper function to replace the first and last passes names by their identifier. */
+private fun mermaidPassName(pass: KClass<out Pass<*>>): String {
+    return when {
+        pass.isFirstPass -> FIRST_PASS_IDENTIFIER
+        pass.isLastPass -> LAST_PASS_IDENTIFIER
+        else -> pass.simpleName ?: UNKNOWN_PASS
+    }
+}
+/**
+ * Builds a markdown representation of a pass dependency graph, based on
+ * [Mermaid](https://mermaid.js.org) syntax.
+ */
+internal fun buildMermaid(passes: List<KClass<out Pass<out Node>>>): String {
+    var s = "```mermaid\n"
+    s += "flowchart TD;\n"
+
+    s += "    subgraph $FIRST_PASSES_SUBGRAPH_IDENTIFIER [\"First Passes\"];\n"
+    passes
+        .filter { it.isFirstPass }
+        .forEach { s += "    $FIRST_PASS_IDENTIFIER[\"${it.simpleName}\"];\n" }
+    s += "    end;\n"
+    s += "    subgraph $LAST_PASSES_SUBGRAPH_IDENTIFIER [\"Last Passes\"];\n"
+    passes
+        .filter { it.isLastPass }
+        .forEach { s += "        $LAST_PASS_IDENTIFIER[\"${it.simpleName}\"];\n" }
+    s += "    end;\n"
+
+    s += "    $FIRST_PASSES_SUBGRAPH_IDENTIFIER~~~$NORMAL_PASSES_SUBGRAPH_IDENTIFIER;\n"
+    s += "    subgraph $NORMAL_PASSES_SUBGRAPH_IDENTIFIER [\"Normal Passes\"];\n"
+    for ((pass, deps) in passes.associateWith { it.softDependencies }.entries) {
+        for (dep in deps) {
+            s += "        ${mermaidPassName(dep)}-.->${mermaidPassName(pass)};\n"
+        }
+    }
+    for ((pass, deps) in passes.associateWith { it.hardDependencies }.entries) {
+        for (dep in deps) {
+            s += "        ${mermaidPassName(dep)}-->${mermaidPassName(pass)};\n"
+        }
+    }
+    for ((pass, before) in passes.associateWith { it.softExecuteBefore }.entries) {
+        for (execBefore in before) {
+            s += "        ${mermaidPassName(pass)}-.->${mermaidPassName(execBefore)};\n"
+        }
+    }
+    for ((pass, beforeList) in passes.associateWith { it.hardExecuteBefore }.entries) {
+        for (execBefore in beforeList) {
+            s += "        ${mermaidPassName(pass)}-->${mermaidPassName(execBefore)};\n"
+        }
+    }
+    s += "    end;\n"
+    s += "    $NORMAL_PASSES_SUBGRAPH_IDENTIFIER~~~$LAST_PASSES_SUBGRAPH_IDENTIFIER;\n"
+    s += "```"
+    return s
+}

cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationConfiguration.kt

@@ -651,171 +651,16 @@ private constructor(
             )
         }
 
-        /**
-         * Collects the requested passes stored in [registeredPasses] and generates a
-         * [PassWithDepsContainer] consisting of pairs of passes and their dependencies.
-         *
-         * @return A populated [PassWithDepsContainer] derived from [registeredPasses].
-         */
-        private fun collectInitialPasses(): PassWithDepsContainer {
-            val workingList = PassWithDepsContainer()
-
-            val softDependencies =
-                mutableMapOf<KClass<out Pass<*>>, MutableSet<KClass<out Pass<*>>>>()
-            val hardDependencies =
-                mutableMapOf<KClass<out Pass<*>>, MutableSet<KClass<out Pass<*>>>>()
-
-            // Add the "execute before" dependencies.
-            for (p in passes) {
-                val executeBefore = mutableListOf<KClass<out Pass<*>>>()
-
-                val depAnn = p.findAnnotations<DependsOn>()
-                // collect all dependencies added by [DependsOn] annotations.
-                for (d in depAnn) {
-                    val deps =
-                        if (d.softDependency) {
-                            softDependencies.computeIfAbsent(p) { mutableSetOf() }
-                        } else {
-                            hardDependencies.computeIfAbsent(p) { mutableSetOf() }
-                        }
-                    deps += d.value
-                }
-
-                val execBeforeAnn = p.findAnnotations<ExecuteBefore>()
-                for (d in execBeforeAnn) {
-                    executeBefore.add(d.other)
-                }
-
-                for (eb in executeBefore) {
-                    passes
-                        .filter { eb == it }
-                        .forEach {
-                            val deps = softDependencies.computeIfAbsent(it) { mutableSetOf() }
-                            deps += p
-                        }
-                }
-            }
-
-            log.info(
-                "The following mermaid graph represents the pass dependencies: \n ${buildMermaid(softDependencies, hardDependencies)}"
-            )
-
-            for (p in passes) {
-                var passFound = false
-                for ((pass) in workingList.getWorkingList()) {
-                    if (pass == p) {
-                        passFound = true
-                        break
-                    }
-                }
-                if (!passFound) {
-                    workingList.addToWorkingList(
-                        PassWithDependencies(
-                            p,
-                            softDependencies[p] ?: mutableSetOf(),
-                            hardDependencies[p] ?: mutableSetOf()
-                        )
-                    )
-                }
-            }
-            return workingList
-        }
-
-        /**
-         * Builds a markdown representation of a pass dependency graph, based on
-         * [Mermaid](https://mermaid.js.org) syntax.
-         */
-        private fun buildMermaid(
-            softDependencies: MutableMap<KClass<out Pass<*>>, MutableSet<KClass<out Pass<*>>>>,
-            hardDependencies: MutableMap<KClass<out Pass<*>>, MutableSet<KClass<out Pass<*>>>>
-        ): String {
-            var s = "```mermaid\n"
-            s += "flowchart TD;\n"
-            for ((pass, deps) in softDependencies.entries) {
-                for (dep in deps) {
-                    s += "    ${dep.simpleName}-->${pass.simpleName};\n"
-                }
-            }
-            for ((pass, deps) in hardDependencies.entries) {
-                for (dep in deps) {
-                    s += "    ${dep.simpleName}-->${pass.simpleName};\n"
-                }
-            }
-            s += "```"
-            return s
-        }
-
-        /**
-         * This function reorders passes in order to meet their dependency requirements.
-         * * soft dependencies [DependsOn] with `softDependency == true`: all passes registered as
-         *   soft dependency will be executed before the current pass if they are registered
-         * * hard dependencies [DependsOn] with `softDependency == false (default)`: all passes
-         *   registered as hard dependency will be executed before the current pass (hard
-         *   dependencies will be registered even if the user did not register them)
-         * * first pass [ExecuteFirst]: a pass registered as first pass will be executed in the
-         *   beginning
-         * * last pass [ExecuteLast]: a pass registered as last pass will be executed at the end
-         *
-         * This function uses a very simple (and inefficient) logic to meet the requirements above:
-         * 1. A list of all registered passes and their dependencies is build
-         *    [PassWithDepsContainer.workingList]
-         * 1. All missing hard dependencies [DependsOn] are added to the
-         *    [PassWithDepsContainer.workingList]
-         * 1. The first pass [ExecuteFirst] is added to the result and removed from the other passes
-         *    dependencies
-         * 1. A list of passes in the workingList without dependencies are added to the result, and
-         *    removed from the other passes dependencies
-         * 1. The above step is repeated until all passes are added to the result
-         *
-         * @return a sorted list of passes, with passes that can be run in parallel together in a
-         *   nested list.
-         */
+        /** This function reorders passes in order to meet their dependency requirements. */
         @Throws(ConfigurationException::class)
         private fun orderPasses(): List<List<KClass<out Pass<*>>>> {
             log.info("Passes before enforcing order: {}", passes.map { it.simpleName })
-            val result = mutableListOf<List<KClass<out Pass<*>>>>()
-
-            // Create a local copy of all passes and their "current" dependencies without possible
-            // duplicates
-            val workingList = collectInitialPasses()
-            log.debug("Working list after initial scan: {}", workingList)
-            workingList.addMissingDependencies()
-            log.debug("Working list after adding missing dependencies: {}", workingList)
-            if (workingList.getFirstPasses().size > 1) {
-                log.error(
-                    "Too many passes require to be executed as first pass: {}",
-                    workingList.getWorkingList()
-                )
-                throw ConfigurationException(
-                    "Too many passes require to be executed as first pass."
-                )
-            }
-            if (workingList.getLastPasses().size > 1) {
-                log.error(
-                    "Too many passes require to be executed as last pass: {}",
-                    workingList.getLastPasses()
-                )
-                throw ConfigurationException("Too many passes require to be executed as last pass.")
-            }
-            val firstPass = workingList.getAndRemoveFirstPass()
-            if (firstPass != null) {
-                result.add(listOf(firstPass))
-            }
-            while (!workingList.isEmpty) {
-                val p = workingList.getAndRemoveFirstPassWithoutDependencies()
-                if (p.isNotEmpty()) {
-                    result.add(p)
-                } else {
-                    // failed to find a pass that can be added to the result -> deadlock :(
-                    throw ConfigurationException("Failed to satisfy ordering requirements.")
-                }
-            }
+            val orderingHelper = PassOrderingHelper(passes)
             log.info(
-                "Passes after enforcing order: {}",
-                result.map { list -> list.map { it.simpleName } }
+                "The following mermaid graph represents the pass dependencies: \n ${buildMermaid(passes)}"
             )
 
-            return result
+            return orderingHelper.order()
         }
     }
 

cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/Pass.kt

@@ -35,15 +35,22 @@ import de.fraunhofer.aisec.cpg.graph.declarations.TranslationUnitDeclaration
 import de.fraunhofer.aisec.cpg.graph.scopes.Scope
 import de.fraunhofer.aisec.cpg.helpers.Benchmark
 import de.fraunhofer.aisec.cpg.helpers.SubgraphWalker.ScopedWalker
+import de.fraunhofer.aisec.cpg.passes.configuration.DependsOn
+import de.fraunhofer.aisec.cpg.passes.configuration.ExecuteBefore
+import de.fraunhofer.aisec.cpg.passes.configuration.ExecuteFirst
+import de.fraunhofer.aisec.cpg.passes.configuration.ExecuteLast
+import de.fraunhofer.aisec.cpg.passes.configuration.ExecuteLate
 import de.fraunhofer.aisec.cpg.passes.configuration.RequiredFrontend
 import de.fraunhofer.aisec.cpg.passes.configuration.RequiresLanguageTrait
 import java.util.concurrent.CompletableFuture
 import java.util.function.Consumer
 import kotlin.reflect.KClass
 import kotlin.reflect.full.findAnnotation
 import kotlin.reflect.full.findAnnotations
+import kotlin.reflect.full.hasAnnotation
 import kotlin.reflect.full.isSubclassOf
 import kotlin.reflect.full.primaryConstructor
+import org.apache.commons.lang3.builder.ToStringBuilder
 import org.slf4j.Logger
 import org.slf4j.LoggerFactory
 
@@ -153,6 +160,47 @@ sealed class Pass<T : Node>(final override val ctx: TranslationContext) :
     fun <T : PassConfiguration> passConfig(): T? {
         return this.config.passConfigurations[this::class] as? T
     }
+
+    override fun toString(): String {
+        val builder =
+            ToStringBuilder(this, Node.TO_STRING_STYLE).append("pass", this::class.simpleName)
+
+        if (this::class.softDependencies.isNotEmpty()) {
+            builder.append("soft dependencies:", this::class.softDependencies.map { it.simpleName })
+        }
+
+        if (this::class.hardDependencies.isNotEmpty()) {
+            builder.append("hard dependencies:", this::class.hardDependencies.map { it.simpleName })
+        }
+
+        if (this::class.softExecuteBefore.isNotEmpty()) {
+            builder.append(
+                "execute before (soft): ",
+                this::class.softExecuteBefore.map { it.simpleName }
+            )
+        }
+
+        if (this::class.hardExecuteBefore.isNotEmpty()) {
+            builder.append(
+                "execute before (hard): ",
+                this::class.hardExecuteBefore.map { it.simpleName }
+            )
+        }
+
+        if (this::class.isFirstPass) {
+            builder.append("firstPass")
+        }
+
+        if (this::class.isLastPass) {
+            builder.append("lastPass")
+        }
+
+        if (this::class.isLatePass) {
+            builder.append("latePass")
+        }
+
+        return builder.toString()
+    }
 }
 
 fun executePassesInParallel(
@@ -318,3 +366,50 @@ fun <T : Node> checkForReplacement(
     @Suppress("UNCHECKED_CAST")
     return config.replacedPasses[Pair(cls, language::class)] as? KClass<out Pass<T>> ?: cls
 }
+
+val KClass<out Pass<*>>.isFirstPass: Boolean
+    get() {
+        return this.hasAnnotation<ExecuteFirst>()
+    }
+
+val KClass<out Pass<*>>.isLastPass: Boolean
+    get() {
+        return this.hasAnnotation<ExecuteLast>()
+    }
+
+val KClass<out Pass<*>>.isLatePass: Boolean
+    get() {
+        return this.hasAnnotation<ExecuteLate>()
+    }
+
+val KClass<out Pass<*>>.softDependencies: Set<KClass<out Pass<*>>>
+    get() {
+        return this.findAnnotations<DependsOn>()
+            .filter { it.softDependency == true }
+            .map { it.value }
+            .toSet()
+    }
+
+val KClass<out Pass<*>>.hardDependencies: Set<KClass<out Pass<*>>>
+    get() {
+        return this.findAnnotations<DependsOn>()
+            .filter { it.softDependency == false }
+            .map { it.value }
+            .toSet()
+    }
+
+val KClass<out Pass<*>>.softExecuteBefore: Set<KClass<out Pass<*>>>
+    get() {
+        return this.findAnnotations<ExecuteBefore>()
+            .filter { it.soft == true }
+            .map { it.other }
+            .toSet()
+    }
+
+val KClass<out Pass<*>>.hardExecuteBefore: Set<KClass<out Pass<*>>>
+    get() {
+        return this.findAnnotations<ExecuteBefore>()
+            .filter { it.soft == false }
+            .map { it.other }
+            .toSet()
+    }