diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/MermaidHelper.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/MermaidHelper.kt new file mode 100644 index 0000000000..61f4d325f8 --- /dev/null +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/MermaidHelper.kt @@ -0,0 +1,98 @@ +/* + * Copyright (c) 2024, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg + +import de.fraunhofer.aisec.cpg.graph.Node +import de.fraunhofer.aisec.cpg.passes.Pass +import de.fraunhofer.aisec.cpg.passes.hardDependencies +import de.fraunhofer.aisec.cpg.passes.hardExecuteBefore +import de.fraunhofer.aisec.cpg.passes.isFirstPass +import de.fraunhofer.aisec.cpg.passes.isLastPass +import de.fraunhofer.aisec.cpg.passes.softDependencies +import de.fraunhofer.aisec.cpg.passes.softExecuteBefore +import kotlin.reflect.KClass + +private const val UNKNOWN_PASS = "UnknownPass" +private const val FIRST_PASSES_SUBGRAPH_IDENTIFIER = "FirstPassesSubgraph" +private const val NORMAL_PASSES_SUBGRAPH_IDENTIFIER = "NormalPassesSubgraph" +private const val LAST_PASSES_SUBGRAPH_IDENTIFIER = "LastPassesSubgraph" +private const val FIRST_PASS_IDENTIFIER = "FirstPass" +private const val LAST_PASS_IDENTIFIER = "LastPass" + +/** Helper function to replace the first and last passes names by their identifier. */ +private fun mermaidPassName(pass: KClass>): String { + return when { + pass.isFirstPass -> FIRST_PASS_IDENTIFIER + pass.isLastPass -> LAST_PASS_IDENTIFIER + else -> pass.simpleName ?: UNKNOWN_PASS + } +} +/** + * Builds a markdown representation of a pass dependency graph, based on + * [Mermaid](https://mermaid.js.org) syntax. + */ +internal fun buildMermaid(passes: List>>): String { + var s = "```mermaid\n" + s += "flowchart TD;\n" + + s += " subgraph $FIRST_PASSES_SUBGRAPH_IDENTIFIER [\"First Passes\"];\n" + passes + .filter { it.isFirstPass } + .forEach { s += " $FIRST_PASS_IDENTIFIER[\"${it.simpleName}\"];\n" } + s += " end;\n" + s += " subgraph $LAST_PASSES_SUBGRAPH_IDENTIFIER [\"Last Passes\"];\n" + passes + .filter { it.isLastPass } + .forEach { s += " $LAST_PASS_IDENTIFIER[\"${it.simpleName}\"];\n" } + s += " end;\n" + + s += " $FIRST_PASSES_SUBGRAPH_IDENTIFIER~~~$NORMAL_PASSES_SUBGRAPH_IDENTIFIER;\n" + s += " subgraph $NORMAL_PASSES_SUBGRAPH_IDENTIFIER [\"Normal Passes\"];\n" + for ((pass, deps) in passes.associateWith { it.softDependencies }.entries) { + for (dep in deps) { + s += " ${mermaidPassName(dep)}-.->${mermaidPassName(pass)};\n" + } + } + for ((pass, deps) in passes.associateWith { it.hardDependencies }.entries) { + for (dep in deps) { + s += " ${mermaidPassName(dep)}-->${mermaidPassName(pass)};\n" + } + } + for ((pass, before) in passes.associateWith { it.softExecuteBefore }.entries) { + for (execBefore in before) { + s += " ${mermaidPassName(pass)}-.->${mermaidPassName(execBefore)};\n" + } + } + for ((pass, beforeList) in passes.associateWith { it.hardExecuteBefore }.entries) { + for (execBefore in beforeList) { + s += " ${mermaidPassName(pass)}-->${mermaidPassName(execBefore)};\n" + } + } + s += " end;\n" + s += " $NORMAL_PASSES_SUBGRAPH_IDENTIFIER~~~$LAST_PASSES_SUBGRAPH_IDENTIFIER;\n" + s += "```" + return s +} diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationConfiguration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationConfiguration.kt index a8daa6b0c9..5d2e481a4b 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationConfiguration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationConfiguration.kt @@ -651,171 +651,16 @@ private constructor( ) } - /** - * Collects the requested passes stored in [registeredPasses] and generates a - * [PassWithDepsContainer] consisting of pairs of passes and their dependencies. - * - * @return A populated [PassWithDepsContainer] derived from [registeredPasses]. - */ - private fun collectInitialPasses(): PassWithDepsContainer { - val workingList = PassWithDepsContainer() - - val softDependencies = - mutableMapOf>, MutableSet>>>() - val hardDependencies = - mutableMapOf>, MutableSet>>>() - - // Add the "execute before" dependencies. - for (p in passes) { - val executeBefore = mutableListOf>>() - - val depAnn = p.findAnnotations() - // collect all dependencies added by [DependsOn] annotations. - for (d in depAnn) { - val deps = - if (d.softDependency) { - softDependencies.computeIfAbsent(p) { mutableSetOf() } - } else { - hardDependencies.computeIfAbsent(p) { mutableSetOf() } - } - deps += d.value - } - - val execBeforeAnn = p.findAnnotations() - for (d in execBeforeAnn) { - executeBefore.add(d.other) - } - - for (eb in executeBefore) { - passes - .filter { eb == it } - .forEach { - val deps = softDependencies.computeIfAbsent(it) { mutableSetOf() } - deps += p - } - } - } - - log.info( - "The following mermaid graph represents the pass dependencies: \n ${buildMermaid(softDependencies, hardDependencies)}" - ) - - for (p in passes) { - var passFound = false - for ((pass) in workingList.getWorkingList()) { - if (pass == p) { - passFound = true - break - } - } - if (!passFound) { - workingList.addToWorkingList( - PassWithDependencies( - p, - softDependencies[p] ?: mutableSetOf(), - hardDependencies[p] ?: mutableSetOf() - ) - ) - } - } - return workingList - } - - /** - * Builds a markdown representation of a pass dependency graph, based on - * [Mermaid](https://mermaid.js.org) syntax. - */ - private fun buildMermaid( - softDependencies: MutableMap>, MutableSet>>>, - hardDependencies: MutableMap>, MutableSet>>> - ): String { - var s = "```mermaid\n" - s += "flowchart TD;\n" - for ((pass, deps) in softDependencies.entries) { - for (dep in deps) { - s += " ${dep.simpleName}-->${pass.simpleName};\n" - } - } - for ((pass, deps) in hardDependencies.entries) { - for (dep in deps) { - s += " ${dep.simpleName}-->${pass.simpleName};\n" - } - } - s += "```" - return s - } - - /** - * This function reorders passes in order to meet their dependency requirements. - * * soft dependencies [DependsOn] with `softDependency == true`: all passes registered as - * soft dependency will be executed before the current pass if they are registered - * * hard dependencies [DependsOn] with `softDependency == false (default)`: all passes - * registered as hard dependency will be executed before the current pass (hard - * dependencies will be registered even if the user did not register them) - * * first pass [ExecuteFirst]: a pass registered as first pass will be executed in the - * beginning - * * last pass [ExecuteLast]: a pass registered as last pass will be executed at the end - * - * This function uses a very simple (and inefficient) logic to meet the requirements above: - * 1. A list of all registered passes and their dependencies is build - * [PassWithDepsContainer.workingList] - * 1. All missing hard dependencies [DependsOn] are added to the - * [PassWithDepsContainer.workingList] - * 1. The first pass [ExecuteFirst] is added to the result and removed from the other passes - * dependencies - * 1. A list of passes in the workingList without dependencies are added to the result, and - * removed from the other passes dependencies - * 1. The above step is repeated until all passes are added to the result - * - * @return a sorted list of passes, with passes that can be run in parallel together in a - * nested list. - */ + /** This function reorders passes in order to meet their dependency requirements. */ @Throws(ConfigurationException::class) private fun orderPasses(): List>>> { log.info("Passes before enforcing order: {}", passes.map { it.simpleName }) - val result = mutableListOf>>>() - - // Create a local copy of all passes and their "current" dependencies without possible - // duplicates - val workingList = collectInitialPasses() - log.debug("Working list after initial scan: {}", workingList) - workingList.addMissingDependencies() - log.debug("Working list after adding missing dependencies: {}", workingList) - if (workingList.getFirstPasses().size > 1) { - log.error( - "Too many passes require to be executed as first pass: {}", - workingList.getWorkingList() - ) - throw ConfigurationException( - "Too many passes require to be executed as first pass." - ) - } - if (workingList.getLastPasses().size > 1) { - log.error( - "Too many passes require to be executed as last pass: {}", - workingList.getLastPasses() - ) - throw ConfigurationException("Too many passes require to be executed as last pass.") - } - val firstPass = workingList.getAndRemoveFirstPass() - if (firstPass != null) { - result.add(listOf(firstPass)) - } - while (!workingList.isEmpty) { - val p = workingList.getAndRemoveFirstPassWithoutDependencies() - if (p.isNotEmpty()) { - result.add(p) - } else { - // failed to find a pass that can be added to the result -> deadlock :( - throw ConfigurationException("Failed to satisfy ordering requirements.") - } - } + val orderingHelper = PassOrderingHelper(passes) log.info( - "Passes after enforcing order: {}", - result.map { list -> list.map { it.simpleName } } + "The following mermaid graph represents the pass dependencies: \n ${buildMermaid(passes)}" ) - return result + return orderingHelper.order() } } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/Pass.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/Pass.kt index 2536b4c64b..350d596221 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/Pass.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/Pass.kt @@ -35,6 +35,11 @@ import de.fraunhofer.aisec.cpg.graph.declarations.TranslationUnitDeclaration import de.fraunhofer.aisec.cpg.graph.scopes.Scope import de.fraunhofer.aisec.cpg.helpers.Benchmark import de.fraunhofer.aisec.cpg.helpers.SubgraphWalker.ScopedWalker +import de.fraunhofer.aisec.cpg.passes.configuration.DependsOn +import de.fraunhofer.aisec.cpg.passes.configuration.ExecuteBefore +import de.fraunhofer.aisec.cpg.passes.configuration.ExecuteFirst +import de.fraunhofer.aisec.cpg.passes.configuration.ExecuteLast +import de.fraunhofer.aisec.cpg.passes.configuration.ExecuteLate import de.fraunhofer.aisec.cpg.passes.configuration.RequiredFrontend import de.fraunhofer.aisec.cpg.passes.configuration.RequiresLanguageTrait import java.util.concurrent.CompletableFuture @@ -42,8 +47,10 @@ import java.util.function.Consumer import kotlin.reflect.KClass import kotlin.reflect.full.findAnnotation import kotlin.reflect.full.findAnnotations +import kotlin.reflect.full.hasAnnotation import kotlin.reflect.full.isSubclassOf import kotlin.reflect.full.primaryConstructor +import org.apache.commons.lang3.builder.ToStringBuilder import org.slf4j.Logger import org.slf4j.LoggerFactory @@ -153,6 +160,47 @@ sealed class Pass(final override val ctx: TranslationContext) : fun passConfig(): T? { return this.config.passConfigurations[this::class] as? T } + + override fun toString(): String { + val builder = + ToStringBuilder(this, Node.TO_STRING_STYLE).append("pass", this::class.simpleName) + + if (this::class.softDependencies.isNotEmpty()) { + builder.append("soft dependencies:", this::class.softDependencies.map { it.simpleName }) + } + + if (this::class.hardDependencies.isNotEmpty()) { + builder.append("hard dependencies:", this::class.hardDependencies.map { it.simpleName }) + } + + if (this::class.softExecuteBefore.isNotEmpty()) { + builder.append( + "execute before (soft): ", + this::class.softExecuteBefore.map { it.simpleName } + ) + } + + if (this::class.hardExecuteBefore.isNotEmpty()) { + builder.append( + "execute before (hard): ", + this::class.hardExecuteBefore.map { it.simpleName } + ) + } + + if (this::class.isFirstPass) { + builder.append("firstPass") + } + + if (this::class.isLastPass) { + builder.append("lastPass") + } + + if (this::class.isLatePass) { + builder.append("latePass") + } + + return builder.toString() + } } fun executePassesInParallel( @@ -318,3 +366,50 @@ fun checkForReplacement( @Suppress("UNCHECKED_CAST") return config.replacedPasses[Pair(cls, language::class)] as? KClass> ?: cls } + +val KClass>.isFirstPass: Boolean + get() { + return this.hasAnnotation() + } + +val KClass>.isLastPass: Boolean + get() { + return this.hasAnnotation() + } + +val KClass>.isLatePass: Boolean + get() { + return this.hasAnnotation() + } + +val KClass>.softDependencies: Set>> + get() { + return this.findAnnotations() + .filter { it.softDependency == true } + .map { it.value } + .toSet() + } + +val KClass>.hardDependencies: Set>> + get() { + return this.findAnnotations() + .filter { it.softDependency == false } + .map { it.value } + .toSet() + } + +val KClass>.softExecuteBefore: Set>> + get() { + return this.findAnnotations() + .filter { it.soft == true } + .map { it.other } + .toSet() + } + +val KClass>.hardExecuteBefore: Set>> + get() { + return this.findAnnotations() + .filter { it.soft == false } + .map { it.other } + .toSet() + } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/PrepareSerialization.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/PrepareSerialization.kt index f61a26239a..7405973bd6 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/PrepareSerialization.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/PrepareSerialization.kt @@ -31,12 +31,12 @@ import de.fraunhofer.aisec.cpg.graph.allChildren import de.fraunhofer.aisec.cpg.graph.declarations.TranslationUnitDeclaration import de.fraunhofer.aisec.cpg.graph.statements.expressions.CallExpression import de.fraunhofer.aisec.cpg.helpers.SubgraphWalker -import de.fraunhofer.aisec.cpg.passes.configuration.ExecuteBefore +import de.fraunhofer.aisec.cpg.passes.configuration.ExecuteLate import kotlin.reflect.full.memberProperties import kotlin.reflect.jvm.javaField /** Pass with some graph transformations useful when doing serialization. */ -@ExecuteBefore(FilenameMapper::class) +@ExecuteLate class PrepareSerialization(ctx: TranslationContext) : TranslationUnitPass(ctx) { private val nodeNameField = Node::class diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/configuration/ExecuteBefore.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/configuration/ExecuteBefore.kt index ab6a5fe07b..959b95bd73 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/configuration/ExecuteBefore.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/configuration/ExecuteBefore.kt @@ -30,9 +30,10 @@ import kotlin.reflect.KClass /** * Register a dependency for the annotated pass. This ensures that the annotated pass is executed - * before [other] pass. + * before [other] pass. The [soft] flag decides whether to treat this as a hard dependency + * (resulting in the pass being registered if not present) or not. */ @Retention(AnnotationRetention.RUNTIME) @Target(AnnotationTarget.CLASS) @Repeatable -annotation class ExecuteBefore(val other: KClass>) +annotation class ExecuteBefore(val other: KClass>, val soft: Boolean = false) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/configuration/ExecuteLate.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/configuration/ExecuteLate.kt new file mode 100644 index 0000000000..ebe46c8c40 --- /dev/null +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/configuration/ExecuteLate.kt @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2024, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.passes.configuration + +/** + * Indicates whether this pass should be executed as late as possible (without breaking any other + * constraints like [ExecuteLast] or [DependsOn], ...) + */ +@Retention(AnnotationRetention.RUNTIME) +@Target(AnnotationTarget.CLASS) +annotation class ExecuteLate diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/configuration/PassOrderingHelper.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/configuration/PassOrderingHelper.kt new file mode 100644 index 0000000000..4baa63f930 --- /dev/null +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/configuration/PassOrderingHelper.kt @@ -0,0 +1,338 @@ +/* + * Copyright (c) 2022, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.passes.configuration + +import de.fraunhofer.aisec.cpg.ConfigurationException +import de.fraunhofer.aisec.cpg.passes.* +import de.fraunhofer.aisec.cpg.passes.Pass +import java.util.* +import kotlin.reflect.KClass +import kotlin.reflect.full.findAnnotations +import org.slf4j.LoggerFactory + +/** + * The goal of this class is to provide ordered passes when invoking the [order] function. + * * soft dependencies ([DependsOn] with `softDependency == true`): all passes registered as soft + * dependency will be executed before the current pass, if they are registered independently + * * hard dependencies ([DependsOn] with `softDependency == false (default)`): all passes registered + * as hard dependency will be executed before the current pass (hard dependencies will be + * registered even if the user did not specifically register them) + * * first pass [ExecuteFirst]: a pass registered as first pass will be executed in the beginning + * * last pass [ExecuteLast]: a pass registered as last pass will be executed at the end + * * late pass [ExecuteLate]: a pass that is executed as late as possible without violating any of + * the other constraints + * * [ExecuteBefore] (with soft and hard dependencies): the pass is to be executed before the other + * pass (soft: if the other pass is also configured) + * + * This class works by + * 1. Setup + * 1. Iterating through the configured passes and registering them (plus all `hard == true` + * dependencies) as [PassWithDependencies] containers: + * 1. [ExecuteFirst] and [ExecuteLast] passes are stored in separate lists to keep the + * ordering logic simple. + * 2. Normal passes are stored in the [workingList] + * 2. [ExecuteBefore] passes: "`A` execute before `B`" implies that "`B` depends on `A`" -> the + * dependency is stored in the [PassWithDependencies.dependenciesRemaining] of "`B`". This + * logic is implemented in [populateExecuteBeforeDependencies]. + * 3. [DependsOn] passes: [PassWithDependencies.dependenciesRemaining] keeps track of these + * dependencies. [populateNormalDependencies] implements this logic. + * 4. A [sanityCheck] is performed. + * 2. Ordering + * 1. [firstPassesList] passes are moved to the result. + * 2. While the [workingList] is not empty: + * 1. All passes ready to be scheduled are moved to the result excluding late passes. If + * this found at least one pass, then the loop starts again. + * 2. Late passes are considered for scheduling as well. If this found at least one pass, + * then the loop starts again. Otherwise, the dependencies cannot be satisfied. + * 3. [lastPassesList] passes are moved to the result. + * + * Note: whenever a pass is moved to the result: + * - it is removed from the [workingList] (and [firstPassesList] / [lastPassesList]) + * - the other pass's [PassWithDependencies.dependenciesRemaining] are updated. + */ +class PassOrderingHelper { + /** This list stores all non-{first,last} passes which have not yet been ordered. */ + private val workingList: MutableList = ArrayList() + + /** This list stores all first passes. Stored separately to keep the sorting logic simpler. */ + private val firstPassesList: MutableList = ArrayList() + + /** This list stores all last passes. Stored separately to keep the sorting logic simpler. */ + private val lastPassesList: MutableList = ArrayList() + + companion object { + private val log = LoggerFactory.getLogger(PassOrderingHelper::class.java) + } + + /** + * Collects the requested passes provided as [passes] and populates the internal [workingList] + * consisting of pairs of passes and their dependencies. Also, this function adds all + * `hardDependencies` + */ + constructor(passes: List>>) { + for (pass in passes) { + addToWorkingList(pass) + } + + // translate "A `executeBefore` B" to "B `dependsOn` A" + populateExecuteBeforeDependencies() + + // clean up soft dependencies which are not registered in the workingList + populateNormalDependencies() + + // finally, run a sanity check + sanityCheck() + } + + /** Register all (soft and hard) dependencies. */ + private fun populateNormalDependencies() { + for (pass in workingList) { + pass.passClass.hardDependencies.forEach { pass.dependenciesRemaining += it } + pass.passClass.softDependencies + .filter { workingList.map { it.passClass }.contains(it) } + .forEach { pass.dependenciesRemaining += it } + } + } + + /** + * Add a pass to the internal [workingList], iff it does not exist. + * + * Also, add + * * hard dependencies + * * [ExecuteBefore] dependencies + */ + private fun addToWorkingList(newElement: KClass>) { + if ( + (workingList + firstPassesList + lastPassesList) + .filter { it.passClass == newElement } + .isNotEmpty() + ) { + // we already know about this pass + return + } + + var firstOrLastPass = false + if (newElement.findAnnotations().isNotEmpty()) { + firstOrLastPass = true + firstPassesList.add(PassWithDependencies(newElement, mutableSetOf())) + } + if (newElement.findAnnotations().isNotEmpty()) { + firstOrLastPass = true + lastPassesList.add(PassWithDependencies(newElement, mutableSetOf())) + } + if (!firstOrLastPass) { + workingList.add(PassWithDependencies(newElement, mutableSetOf())) + } + + // take care of hard dependencies + for (dep in newElement.findAnnotations()) { + if (!dep.softDependency) { // only hard dependencies + addToWorkingList(dep.value) + } + } + + // take care of [ExecuteBefore] dependencies + for (dep in newElement.findAnnotations()) { + if (!dep.soft) { + addToWorkingList(dep.other) + } + } + } + + /** + * Order the passes. This function honors + * - [DependsOn] with soft and hard dependencies + * - [ExecuteFirst] + * - [ExecuteLast] + * - [ExecuteLate] + * - [ExecuteBefore] with soft and hard dependencies + * + * @return a sorted list of passes, with passes that can be run in parallel together in a nested + * list. + * @throws [ConfigurationException] if the passes cannot be ordered. + */ + fun order(): List>>> { + val result = mutableListOf>>>() + + // [ExecuteFirst] + getAndRemoveFirstPasses()?.let { + result.add(listOf(it)) + } // there can only be one because of [sanityCheck] + + // "normal / middle" passes + while (workingList.isNotEmpty()) { + val noLatePassesAllowed = getAndRemoveNextPasses(allowLatePasses = false) + if (noLatePassesAllowed.isNotEmpty()) { + result.add(noLatePassesAllowed) + } else { + val latePassesAllowed = getAndRemoveNextPasses(allowLatePasses = true) + if (latePassesAllowed.isNotEmpty()) { + result.add(latePassesAllowed) + } else { + throw ConfigurationException("Failed to satisfy ordering requirements.") + } + } + } + + // [ExecuteLast] + lastPassesList.firstOrNull()?.let { + result.add(listOf(selectPass(it))) + } // there can only be one because of [sanityCheck] + + log.info( + "Passes after enforcing order: {}", + result.map { list -> list.map { it.simpleName } } + ) + return result + } + + /** + * A pass annotated with [ExecuteBefore] implies that the other pass depends on it. We populate + * the [de.fraunhofer.aisec.cpg.passes.configuration.PassWithDependencies.dependenciesRemaining] + * field in the other pass to make the analysis simpler. + */ + private fun populateExecuteBeforeDependencies() { + for (pass in + (workingList + firstPassesList + lastPassesList)) { // iterate over entire workingList + for (executeBeforePass in + (pass.passClass.softExecuteBefore + + pass.passClass.hardExecuteBefore)) { // iterate over all executeBefore passes + (workingList + firstPassesList + lastPassesList) + .map { it } + .filter { it.passClass == executeBeforePass } // find the executeBeforePass + .forEach { + it.dependenciesRemaining += pass.passClass + } // add the original pass to the dependency list + } + } + } + + /** + * Iterate through all elements and remove the provided dependency [cls] from all passes in the + * working lists. + */ + private fun removeDependencyByClass(cls: KClass>) { + for (pass in workingList) { + pass.dependenciesRemaining.remove(cls) + } + for (pass in firstPassesList) { + pass.dependenciesRemaining.remove(cls) + } + for (pass in lastPassesList) { + pass.dependenciesRemaining.remove(cls) + } + } + + /** + * Finds the first passes which have all their dependencies satisfied. These passes are then + * returned. + * + * @return The first passes which have no active dependencies on success. An empty list + * otherwise. + */ + private fun getAndRemoveNextPasses(allowLatePasses: Boolean): List>> { + return workingList + .filter { + it.dependenciesRemaining.isEmpty() && it.passClass.isLatePass == allowLatePasses + } + .map { selectPass(it) } + } + + /** + * Checks for passes marked as first pass by [ExecuteFirst] + * + * If found, this pass is returned and removed from the working list. + * + * @return The first pass if present. Otherwise, null. + */ + private fun getAndRemoveFirstPasses(): KClass>? { + return when (firstPassesList.isEmpty()) { + true -> null + false -> selectPass(firstPassesList.first()) + } + } + + /** + * Removes a pass from the other passes dependencies and the workingList. + * + * @return the (unpacked) pass + */ + private fun selectPass(pass: PassWithDependencies): KClass> { + // remove it from the other passes dependencies + removeDependencyByClass(pass.passClass) + + // remove it from the workingList + workingList.remove(pass) + firstPassesList.remove(pass) + lastPassesList.remove(pass) + + // return the pass (not the [PassWithDependencies] container) + return pass.passClass + } + + /** + * Perform a sanity check on the configured [workingList]. Currently, this only checks that + * * there is at most one [ExecuteFirst] and + * * at most one [ExecuteLast] pass configured and + * * the first pass does not have a hard dependency and + * * the last pass is not to be executed before other passes. + * + * This does not check, whether the requested ordering can be satisfied. + */ + private fun sanityCheck() { + if (firstPassesList.size > 1) { + throw ConfigurationException( + "More than one pass registered as first pass: \"${firstPassesList.map { it.passClass }}\"." + ) + } + if (lastPassesList.size > 1) { + throw ConfigurationException( + "More than one pass registered as last pass: \"${lastPassesList.map { it.passClass }}\"." + ) + } + + firstPassesList.map { firstPass -> + if (firstPass.passClass.hardDependencies.isNotEmpty()) { + throw ConfigurationException( + "The first pass \"${firstPass.passClass}\" has a hard dependency: \"${firstPass.passClass.hardDependencies}\"." + ) + } + } + + lastPassesList.map { lastPass -> + if (lastPass.passClass.softExecuteBefore.isNotEmpty()) { + throw ConfigurationException( + "The last pass \"${lastPass.passClass}\" is supposed to be executed before another pass: \"${lastPass.passClass.softExecuteBefore}\"." + ) + } + if (lastPass.passClass.hardExecuteBefore.isNotEmpty()) { + throw ConfigurationException( + "The last pass \"${lastPass.passClass}\" is supposed to be executed before another pass: \"${lastPass.passClass.hardExecuteBefore}\"." + ) + } + } + } +} diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/configuration/PassWithDependencies.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/configuration/PassWithDependencies.kt index 50ff80b028..18044c6104 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/configuration/PassWithDependencies.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/configuration/PassWithDependencies.kt @@ -25,71 +25,17 @@ */ package de.fraunhofer.aisec.cpg.passes.configuration -import de.fraunhofer.aisec.cpg.graph.Node import de.fraunhofer.aisec.cpg.passes.Pass import kotlin.reflect.KClass -import kotlin.reflect.full.hasAnnotation -import org.apache.commons.lang3.builder.ToStringBuilder -/** A simple helper class to match a pass with dependencies. */ +/** + * A simple helper class to match a pass with its dependencies. [dependenciesRemaining] shows the + * currently remaining / unsatisfied dependencies. These values are updated during the ordering + * procedure. + */ data class PassWithDependencies( - val pass: KClass>, - val softDependencies: MutableSet>>, - val hardDependencies: MutableSet>> -) { - val dependencies: Set>> - get() { - return softDependencies + hardDependencies - } - - val isFirstPass: Boolean - get() { - return pass.hasAnnotation() - } - - val isLastPass: Boolean - get() { - return pass.hasAnnotation() - } - - override fun toString(): String { - val builder = ToStringBuilder(this, Node.TO_STRING_STYLE).append("pass", pass.simpleName) - - if (softDependencies.isNotEmpty()) { - builder.append("softDependencies", softDependencies.map { it.simpleName }) - } - - if (hardDependencies.isNotEmpty()) { - builder.append("hardDependencies", hardDependencies.map { it.simpleName }) - } - return builder.toString() - } - - /** - * Checks whether the [dependencies] of this pass are met. The list of [softDependencies] and - * [hardDependencies] is removed step-by-step in - * [PassWithDepsContainer.getAndRemoveFirstPassWithoutDependencies]. - */ - fun dependenciesMet(workingList: MutableList): Boolean { - // In the simplest case all our dependencies are empty since they were already removed by - // the selecting algorithm. - if (this.dependencies.isEmpty() && !this.isLastPass) { - return true - } - - // We also need to check, whether we still "soft" depend on passes that are just not - // there (after all hard dependencies are met), in this case we can select the pass - // as well - val remainingClasses = workingList.map { it.pass } - if ( - this.hardDependencies.isEmpty() && - this.softDependencies.all { !remainingClasses.contains(it) } && - !this.isLastPass - ) { - return true - } - - // Otherwise, we still depend on an unselected pass - return false - } -} + /** the pass itself */ + val passClass: KClass>, + /** currently unsatisfied dependencies (soft / hard / [ExecuteBefore] from other passes) */ + val dependenciesRemaining: MutableSet>> +) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/configuration/PassWithDepsContainer.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/configuration/PassWithDepsContainer.kt deleted file mode 100644 index c3219e55c2..0000000000 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/configuration/PassWithDepsContainer.kt +++ /dev/null @@ -1,208 +0,0 @@ -/* - * Copyright (c) 2022, Fraunhofer AISEC. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * $$$$$$\ $$$$$$$\ $$$$$$\ - * $$ __$$\ $$ __$$\ $$ __$$\ - * $$ / \__|$$ | $$ |$$ / \__| - * $$ | $$$$$$$ |$$ |$$$$\ - * $$ | $$ ____/ $$ |\_$$ | - * $$ | $$\ $$ | $$ | $$ | - * \$$$$$ |$$ | \$$$$$ | - * \______/ \__| \______/ - * - */ -package de.fraunhofer.aisec.cpg.passes.configuration - -import de.fraunhofer.aisec.cpg.ConfigurationException -import de.fraunhofer.aisec.cpg.passes.Pass -import de.fraunhofer.aisec.cpg.passes.Pass.Companion.log -import java.util.* -import kotlin.reflect.KClass -import kotlin.reflect.full.findAnnotations - -/** - * A simple helper class for keeping track of passes and their (currently not satisfied) - * dependencies during ordering. - */ -class PassWithDepsContainer { - private val workingList: MutableList - - init { - workingList = ArrayList() - } - - fun getWorkingList(): List { - return workingList - } - - fun addToWorkingList(newElement: PassWithDependencies) { - workingList.add(newElement) - } - - val isEmpty: Boolean - get() = workingList.isEmpty() - - fun size(): Int { - return workingList.size - } - - /** - * Iterate through all elements and remove the provided dependency [cls] from all passes in the - * working list. - */ - private fun removeDependencyByClass(cls: KClass>) { - for (pass in workingList) { - pass.softDependencies.remove(cls) - pass.hardDependencies.remove(cls) - } - } - - override fun toString(): String { - return workingList.toString() - } - - fun getFirstPasses(): List { - return workingList.filter { it.isFirstPass } - } - - fun getLastPasses(): List { - return workingList.filter { it.isLastPass } - } - - private fun dependencyPresent(dep: KClass>): Boolean { - var result = false - for (currentElement in workingList) { - if (dep == currentElement.pass) { - result = true - break - } - } - - return result - } - - private fun createNewPassWithDependency(cls: KClass>): PassWithDependencies { - val softDependencies = mutableSetOf>>() - val hardDependencies = mutableSetOf>>() - - val dependencies = cls.findAnnotations() - for (d in dependencies) { - if (d.softDependency) { - softDependencies += d.value - } else { - hardDependencies += d.value - } - } - - return PassWithDependencies(cls, softDependencies, hardDependencies) - } - - /** - * Recursively iterates the workingList and adds all hard dependencies [DependsOn] and their - * dependencies to the workingList. - */ - fun addMissingDependencies() { - val it = workingList.listIterator() - while (it.hasNext()) { - val current = it.next() - for (dependency in current.hardDependencies) { - if (!dependencyPresent(dependency)) { - log.info( - "Registering a required hard dependency which was not registered explicitly: {}", - dependency - ) - it.add(createNewPassWithDependency(dependency)) - } - } - } - - // add required dependencies to the working list - val missingPasses: MutableList>> = ArrayList() - - // initially populate the missing dependencies list given the current passes - for (currentElement in workingList) { - for (dependency in currentElement.hardDependencies) { - if (!dependencyPresent(dependency)) { - missingPasses.add(dependency) - } - } - } - } - - /** - * Finds the first pass that has all its dependencies satisfied. This pass is then removed from - * the other passes dependencies and returned. - * - * @return The first pass that has no active dependencies on success. null otherwise. - */ - fun getAndRemoveFirstPassWithoutDependencies(): List>> { - val results = mutableListOf>>() - val it = workingList.listIterator() - - while (it.hasNext()) { - val currentElement = it.next() - if (results.isEmpty() && currentElement.isLastPass && workingList.size == 1) { - it.remove() - return listOf(currentElement.pass) - } - - // Keep going until our dependencies are met, this will collect passes that can run in - // parallel in results - if (currentElement.dependenciesMet(workingList)) { - // no unsatisfied dependencies - val result = currentElement.pass - results.add(result) - - // remove pass from the work-list - it.remove() - } else { - continue - } - } - - // remove the selected passes from the other pass's dependencies - results.forEach { removeDependencyByClass(it) } - - return results - } - - /** - * Checks for passes marked as first pass by [ExecuteFirst] - * - * If found, this pass is returned and removed from the working list. - * - * @return The first pass if present. Otherwise, null. - */ - fun getAndRemoveFirstPass(): KClass>? { - val firstPasses = getFirstPasses() - if (firstPasses.size > 1) { - throw ConfigurationException( - "More than one pass requires to be run as first pass: {}".format(firstPasses) - ) - } - return if (firstPasses.isNotEmpty()) { - val firstPass = firstPasses.first() - if (firstPass.hardDependencies.isNotEmpty()) { - throw ConfigurationException("The first pass has a hard dependency.") - } else { - removeDependencyByClass(firstPass.pass) - workingList.remove(firstPass) - firstPass.pass - } - } else { - null - } - } -}