Skip to content

Commit

Permalink
Improve the Pass system (#1645)
Browse files Browse the repository at this point in the history
  • Loading branch information
@maximiliankaul
maximiliankaul authored Sep 5, 2024
1 parent def570f commit e36b64d
Show file tree
Hide file tree
Showing 9 changed files with 584 additions and 435 deletions.
98 changes: 98 additions & 0 deletions cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/MermaidHelper.kt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
/*
* Copyright (c) 2024, Fraunhofer AISEC. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* $$$$$$\ $$$$$$$\ $$$$$$\
* $$ __$$\ $$ __$$\ $$ __$$\
* $$ / \__|$$ | $$ |$$ / \__|
* $$ | $$$$$$$ |$$ |$$$$\
* $$ | $$ ____/ $$ |\_$$ |
* $$ | $$\ $$ | $$ | $$ |
* \$$$$$ |$$ | \$$$$$ |
* \______/ \__| \______/
*
*/
package de.fraunhofer.aisec.cpg

import de.fraunhofer.aisec.cpg.graph.Node
import de.fraunhofer.aisec.cpg.passes.Pass
import de.fraunhofer.aisec.cpg.passes.hardDependencies
import de.fraunhofer.aisec.cpg.passes.hardExecuteBefore
import de.fraunhofer.aisec.cpg.passes.isFirstPass
import de.fraunhofer.aisec.cpg.passes.isLastPass
import de.fraunhofer.aisec.cpg.passes.softDependencies
import de.fraunhofer.aisec.cpg.passes.softExecuteBefore
import kotlin.reflect.KClass

private const val UNKNOWN_PASS = "UnknownPass"
private const val FIRST_PASSES_SUBGRAPH_IDENTIFIER = "FirstPassesSubgraph"
private const val NORMAL_PASSES_SUBGRAPH_IDENTIFIER = "NormalPassesSubgraph"
private const val LAST_PASSES_SUBGRAPH_IDENTIFIER = "LastPassesSubgraph"
private const val FIRST_PASS_IDENTIFIER = "FirstPass"
private const val LAST_PASS_IDENTIFIER = "LastPass"

/** Helper function to replace the first and last passes names by their identifier. */
private fun mermaidPassName(pass: KClass<out Pass<*>>): String {
return when {
pass.isFirstPass -> FIRST_PASS_IDENTIFIER
pass.isLastPass -> LAST_PASS_IDENTIFIER
else -> pass.simpleName ?: UNKNOWN_PASS
}
}
/**
* Builds a markdown representation of a pass dependency graph, based on
* [Mermaid](https://mermaid.js.org) syntax.
*/
internal fun buildMermaid(passes: List<KClass<out Pass<out Node>>>): String {
var s = "```mermaid\n"
s += "flowchart TD;\n"

s += " subgraph $FIRST_PASSES_SUBGRAPH_IDENTIFIER [\"First Passes\"];\n"
passes
.filter { it.isFirstPass }
.forEach { s += " $FIRST_PASS_IDENTIFIER[\"${it.simpleName}\"];\n" }
s += " end;\n"
s += " subgraph $LAST_PASSES_SUBGRAPH_IDENTIFIER [\"Last Passes\"];\n"
passes
.filter { it.isLastPass }
.forEach { s += " $LAST_PASS_IDENTIFIER[\"${it.simpleName}\"];\n" }
s += " end;\n"

s += " $FIRST_PASSES_SUBGRAPH_IDENTIFIER~~~$NORMAL_PASSES_SUBGRAPH_IDENTIFIER;\n"
s += " subgraph $NORMAL_PASSES_SUBGRAPH_IDENTIFIER [\"Normal Passes\"];\n"
for ((pass, deps) in passes.associateWith { it.softDependencies }.entries) {
for (dep in deps) {
s += " ${mermaidPassName(dep)}-.->${mermaidPassName(pass)};\n"
}
}
for ((pass, deps) in passes.associateWith { it.hardDependencies }.entries) {
for (dep in deps) {
s += " ${mermaidPassName(dep)}-->${mermaidPassName(pass)};\n"
}
}
for ((pass, before) in passes.associateWith { it.softExecuteBefore }.entries) {
for (execBefore in before) {
s += " ${mermaidPassName(pass)}-.->${mermaidPassName(execBefore)};\n"
}
}
for ((pass, beforeList) in passes.associateWith { it.hardExecuteBefore }.entries) {
for (execBefore in beforeList) {
s += " ${mermaidPassName(pass)}-->${mermaidPassName(execBefore)};\n"
}
}
s += " end;\n"
s += " $NORMAL_PASSES_SUBGRAPH_IDENTIFIER~~~$LAST_PASSES_SUBGRAPH_IDENTIFIER;\n"
s += "```"
return s
}
163 changes: 4 additions & 159 deletions cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationConfiguration.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -651,171 +651,16 @@ private constructor(
)
}

/**
* Collects the requested passes stored in [registeredPasses] and generates a
* [PassWithDepsContainer] consisting of pairs of passes and their dependencies.
*
* @return A populated [PassWithDepsContainer] derived from [registeredPasses].
*/
private fun collectInitialPasses(): PassWithDepsContainer {
val workingList = PassWithDepsContainer()

val softDependencies =
mutableMapOf<KClass<out Pass<*>>, MutableSet<KClass<out Pass<*>>>>()
val hardDependencies =
mutableMapOf<KClass<out Pass<*>>, MutableSet<KClass<out Pass<*>>>>()

// Add the "execute before" dependencies.
for (p in passes) {
val executeBefore = mutableListOf<KClass<out Pass<*>>>()

val depAnn = p.findAnnotations<DependsOn>()
// collect all dependencies added by [DependsOn] annotations.
for (d in depAnn) {
val deps =
if (d.softDependency) {
softDependencies.computeIfAbsent(p) { mutableSetOf() }
} else {
hardDependencies.computeIfAbsent(p) { mutableSetOf() }
}
deps += d.value
}

val execBeforeAnn = p.findAnnotations<ExecuteBefore>()
for (d in execBeforeAnn) {
executeBefore.add(d.other)
}

for (eb in executeBefore) {
passes
.filter { eb == it }
.forEach {
val deps = softDependencies.computeIfAbsent(it) { mutableSetOf() }
deps += p
}
}
}

log.info(
"The following mermaid graph represents the pass dependencies: \n ${buildMermaid(softDependencies, hardDependencies)}"
)

for (p in passes) {
var passFound = false
for ((pass) in workingList.getWorkingList()) {
if (pass == p) {
passFound = true
break
}
}
if (!passFound) {
workingList.addToWorkingList(
PassWithDependencies(
p,
softDependencies[p] ?: mutableSetOf(),
hardDependencies[p] ?: mutableSetOf()
)
)
}
}
return workingList
}

/**
* Builds a markdown representation of a pass dependency graph, based on
* [Mermaid](https://mermaid.js.org) syntax.
*/
private fun buildMermaid(
softDependencies: MutableMap<KClass<out Pass<*>>, MutableSet<KClass<out Pass<*>>>>,
hardDependencies: MutableMap<KClass<out Pass<*>>, MutableSet<KClass<out Pass<*>>>>
): String {
var s = "```mermaid\n"
s += "flowchart TD;\n"
for ((pass, deps) in softDependencies.entries) {
for (dep in deps) {
s += " ${dep.simpleName}-->${pass.simpleName};\n"
}
}
for ((pass, deps) in hardDependencies.entries) {
for (dep in deps) {
s += " ${dep.simpleName}-->${pass.simpleName};\n"
}
}
s += "```"
return s
}

/**
* This function reorders passes in order to meet their dependency requirements.
* * soft dependencies [DependsOn] with `softDependency == true`: all passes registered as
* soft dependency will be executed before the current pass if they are registered
* * hard dependencies [DependsOn] with `softDependency == false (default)`: all passes
* registered as hard dependency will be executed before the current pass (hard
* dependencies will be registered even if the user did not register them)
* * first pass [ExecuteFirst]: a pass registered as first pass will be executed in the
* beginning
* * last pass [ExecuteLast]: a pass registered as last pass will be executed at the end
*
* This function uses a very simple (and inefficient) logic to meet the requirements above:
* 1. A list of all registered passes and their dependencies is build
* [PassWithDepsContainer.workingList]
* 1. All missing hard dependencies [DependsOn] are added to the
* [PassWithDepsContainer.workingList]
* 1. The first pass [ExecuteFirst] is added to the result and removed from the other passes
* dependencies
* 1. A list of passes in the workingList without dependencies are added to the result, and
* removed from the other passes dependencies
* 1. The above step is repeated until all passes are added to the result
*
* @return a sorted list of passes, with passes that can be run in parallel together in a
* nested list.
*/
/** This function reorders passes in order to meet their dependency requirements. */
@Throws(ConfigurationException::class)
private fun orderPasses(): List<List<KClass<out Pass<*>>>> {
log.info("Passes before enforcing order: {}", passes.map { it.simpleName })
val result = mutableListOf<List<KClass<out Pass<*>>>>()

// Create a local copy of all passes and their "current" dependencies without possible
// duplicates
val workingList = collectInitialPasses()
log.debug("Working list after initial scan: {}", workingList)
workingList.addMissingDependencies()
log.debug("Working list after adding missing dependencies: {}", workingList)
if (workingList.getFirstPasses().size > 1) {
log.error(
"Too many passes require to be executed as first pass: {}",
workingList.getWorkingList()
)
throw ConfigurationException(
"Too many passes require to be executed as first pass."
)
}
if (workingList.getLastPasses().size > 1) {
log.error(
"Too many passes require to be executed as last pass: {}",
workingList.getLastPasses()
)
throw ConfigurationException("Too many passes require to be executed as last pass.")
}
val firstPass = workingList.getAndRemoveFirstPass()
if (firstPass != null) {
result.add(listOf(firstPass))
}
while (!workingList.isEmpty) {
val p = workingList.getAndRemoveFirstPassWithoutDependencies()
if (p.isNotEmpty()) {
result.add(p)
} else {
// failed to find a pass that can be added to the result -> deadlock :(
throw ConfigurationException("Failed to satisfy ordering requirements.")
}
}
val orderingHelper = PassOrderingHelper(passes)
log.info(
"Passes after enforcing order: {}",
result.map { list -> list.map { it.simpleName } }
"The following mermaid graph represents the pass dependencies: \n ${buildMermaid(passes)}"
)

return result
return orderingHelper.order()
}
}

Expand Down
95 changes: 95 additions & 0 deletions cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/Pass.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,15 +35,22 @@ import de.fraunhofer.aisec.cpg.graph.declarations.TranslationUnitDeclaration
import de.fraunhofer.aisec.cpg.graph.scopes.Scope
import de.fraunhofer.aisec.cpg.helpers.Benchmark
import de.fraunhofer.aisec.cpg.helpers.SubgraphWalker.ScopedWalker
import de.fraunhofer.aisec.cpg.passes.configuration.DependsOn
import de.fraunhofer.aisec.cpg.passes.configuration.ExecuteBefore
import de.fraunhofer.aisec.cpg.passes.configuration.ExecuteFirst
import de.fraunhofer.aisec.cpg.passes.configuration.ExecuteLast
import de.fraunhofer.aisec.cpg.passes.configuration.ExecuteLate
import de.fraunhofer.aisec.cpg.passes.configuration.RequiredFrontend
import de.fraunhofer.aisec.cpg.passes.configuration.RequiresLanguageTrait
import java.util.concurrent.CompletableFuture
import java.util.function.Consumer
import kotlin.reflect.KClass
import kotlin.reflect.full.findAnnotation
import kotlin.reflect.full.findAnnotations
import kotlin.reflect.full.hasAnnotation
import kotlin.reflect.full.isSubclassOf
import kotlin.reflect.full.primaryConstructor
import org.apache.commons.lang3.builder.ToStringBuilder
import org.slf4j.Logger
import org.slf4j.LoggerFactory

Expand Down Expand Up @@ -153,6 +160,47 @@ sealed class Pass<T : Node>(final override val ctx: TranslationContext) :
fun <T : PassConfiguration> passConfig(): T? {
return this.config.passConfigurations[this::class] as? T
}

override fun toString(): String {
val builder =
ToStringBuilder(this, Node.TO_STRING_STYLE).append("pass", this::class.simpleName)

if (this::class.softDependencies.isNotEmpty()) {
builder.append("soft dependencies:", this::class.softDependencies.map { it.simpleName })
}

if (this::class.hardDependencies.isNotEmpty()) {
builder.append("hard dependencies:", this::class.hardDependencies.map { it.simpleName })
}

if (this::class.softExecuteBefore.isNotEmpty()) {
builder.append(
"execute before (soft): ",
this::class.softExecuteBefore.map { it.simpleName }
)
}

if (this::class.hardExecuteBefore.isNotEmpty()) {
builder.append(
"execute before (hard): ",
this::class.hardExecuteBefore.map { it.simpleName }
)
}

if (this::class.isFirstPass) {
builder.append("firstPass")
}

if (this::class.isLastPass) {
builder.append("lastPass")
}

if (this::class.isLatePass) {
builder.append("latePass")
}

return builder.toString()
}
}

fun executePassesInParallel(
Expand Down Expand Up @@ -318,3 +366,50 @@ fun <T : Node> checkForReplacement(
@Suppress("UNCHECKED_CAST")
return config.replacedPasses[Pair(cls, language::class)] as? KClass<out Pass<T>> ?: cls
}

val KClass<out Pass<*>>.isFirstPass: Boolean
get() {
return this.hasAnnotation<ExecuteFirst>()
}

val KClass<out Pass<*>>.isLastPass: Boolean
get() {
return this.hasAnnotation<ExecuteLast>()
}

val KClass<out Pass<*>>.isLatePass: Boolean
get() {
return this.hasAnnotation<ExecuteLate>()
}

val KClass<out Pass<*>>.softDependencies: Set<KClass<out Pass<*>>>
get() {
return this.findAnnotations<DependsOn>()
.filter { it.softDependency == true }
.map { it.value }
.toSet()
}

val KClass<out Pass<*>>.hardDependencies: Set<KClass<out Pass<*>>>
get() {
return this.findAnnotations<DependsOn>()
.filter { it.softDependency == false }
.map { it.value }
.toSet()
}

val KClass<out Pass<*>>.softExecuteBefore: Set<KClass<out Pass<*>>>
get() {
return this.findAnnotations<ExecuteBefore>()
.filter { it.soft == true }
.map { it.other }
.toSet()
}

val KClass<out Pass<*>>.hardExecuteBefore: Set<KClass<out Pass<*>>>
get() {
return this.findAnnotations<ExecuteBefore>()
.filter { it.soft == false }
.map { it.other }
.toSet()
}
Loading

0 comments on commit e36b64d

Please sign in to comment.