diff --git a/cpg-core/src/main/java/de/fraunhofer/aisec/cpg/InferenceConfiguration.kt b/cpg-core/src/main/java/de/fraunhofer/aisec/cpg/InferenceConfiguration.kt index 176c75ba35..11c7dc6925 100644 --- a/cpg-core/src/main/java/de/fraunhofer/aisec/cpg/InferenceConfiguration.kt +++ b/cpg-core/src/main/java/de/fraunhofer/aisec/cpg/InferenceConfiguration.kt @@ -26,6 +26,8 @@ package de.fraunhofer.aisec.cpg import de.fraunhofer.aisec.cpg.frontends.cpp.CXXLanguageFrontend +import org.apache.commons.lang3.builder.ToStringBuilder +import org.apache.commons.lang3.builder.ToStringStyle /** * This class holds configuration options for the inference of certain constructs and auto-guessing @@ -51,4 +53,11 @@ private constructor( return Builder() } } + + override fun toString(): String { + return ToStringBuilder(this, ToStringStyle.JSON_STYLE) + .append("guessCastExpressions", guessCastExpressions) + .append("inferRecords", inferRecords) + .toString() + } } diff --git a/cpg-core/src/main/java/de/fraunhofer/aisec/cpg/TranslationConfiguration.java b/cpg-core/src/main/java/de/fraunhofer/aisec/cpg/TranslationConfiguration.java index 18ee70d1f4..83d3b1d6b1 100644 --- a/cpg-core/src/main/java/de/fraunhofer/aisec/cpg/TranslationConfiguration.java +++ b/cpg-core/src/main/java/de/fraunhofer/aisec/cpg/TranslationConfiguration.java @@ -38,6 +38,8 @@ import java.io.File; import java.lang.reflect.InvocationTargetException; import java.util.*; +import org.apache.commons.lang3.builder.ToStringBuilder; +import org.apache.commons.lang3.builder.ToStringStyle; import org.checkerframework.checker.nullness.qual.NonNull; import org.checkerframework.checker.nullness.qual.Nullable; import org.slf4j.Logger; @@ -552,4 +554,28 @@ public TranslationConfiguration build() { compilationDatabase); } } + + @Override + public String toString() { + return new ToStringBuilder(this, ToStringStyle.JSON_STYLE) + .append("debugParser", debugParser) + .append("loadIncludes", loadIncludes) + .append("includePaths", includePaths) + .append("includeWhitelist", includeWhitelist) + .append("includeBlacklist", includeBlacklist) + .append("frontends", frontends) + .append("disableCleanup", disableCleanup) + .append("codeInNodes", codeInNodes) + .append("processAnnotations", processAnnotations) + .append("failOnError", failOnError) + .append("symbols", symbols) + .append("sourceLocations", sourceLocations) + .append("topLevel", topLevel) + .append("useUnityBuild", useUnityBuild) + .append("useParallelFrontends", useParallelFrontends) + .append("typeSystemActiveInFrontend", typeSystemActiveInFrontend) + .append("passes", passes) + .append("inferenceConfiguration", inferenceConfiguration) + .toString(); + } } diff --git a/cpg-core/src/main/java/de/fraunhofer/aisec/cpg/TranslationManager.kt b/cpg-core/src/main/java/de/fraunhofer/aisec/cpg/TranslationManager.kt index c03ac2da5d..9e188849dc 100644 --- a/cpg-core/src/main/java/de/fraunhofer/aisec/cpg/TranslationManager.kt +++ b/cpg-core/src/main/java/de/fraunhofer/aisec/cpg/TranslationManager.kt @@ -76,13 +76,18 @@ private constructor( return CompletableFuture.supplyAsync { val scopesBuildForAnalysis = ScopeManager() val outerBench = - Benchmark(TranslationManager::class.java, "Translation into full graph") + Benchmark( + TranslationManager::class.java, + "Translation into full graph", + false, + result + ) val passesNeedCleanup = mutableSetOf() var frontendsNeedCleanup: Set? = null try { // Parse Java/C/CPP files - var bench = Benchmark(this.javaClass, "Frontend") + var bench = Benchmark(this.javaClass, "Executing Language Frontend", false, result) frontendsNeedCleanup = runFrontends(result, config, scopesBuildForAnalysis) bench.stop() @@ -92,7 +97,7 @@ private constructor( // Apply passes for (pass in config.registeredPasses) { passesNeedCleanup.add(pass) - bench = Benchmark(pass.javaClass, "Executing Pass") + bench = Benchmark(pass.javaClass, "Executing Pass", false, result) pass.accept(result) bench.stop() if (result.isCancelled) { diff --git a/cpg-core/src/main/java/de/fraunhofer/aisec/cpg/TranslationResult.java b/cpg-core/src/main/java/de/fraunhofer/aisec/cpg/TranslationResult.java index f55963dd44..66781da69b 100644 --- a/cpg-core/src/main/java/de/fraunhofer/aisec/cpg/TranslationResult.java +++ b/cpg-core/src/main/java/de/fraunhofer/aisec/cpg/TranslationResult.java @@ -28,18 +28,22 @@ import de.fraunhofer.aisec.cpg.graph.Node; import de.fraunhofer.aisec.cpg.graph.SubGraph; import de.fraunhofer.aisec.cpg.graph.declarations.TranslationUnitDeclaration; +import de.fraunhofer.aisec.cpg.helpers.Benchmark; +import de.fraunhofer.aisec.cpg.helpers.StatisticsHolder; import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; +import org.jetbrains.annotations.NotNull; /** * The global (intermediate) result of the translation. A {@link * de.fraunhofer.aisec.cpg.frontends.LanguageFrontend} will initially populate it and a {@link * de.fraunhofer.aisec.cpg.passes.Pass} can extend it. */ -public class TranslationResult extends Node { +public class TranslationResult extends Node implements StatisticsHolder { public static final String SOURCE_LOCATIONS_TO_FRONTEND = "sourceLocationsToFrontend"; private final TranslationManager translationManager; @@ -50,6 +54,8 @@ public class TranslationResult extends Node { /** A free-for-use HashMap where passes can store whatever they want. */ private final Map scratch = new ConcurrentHashMap<>(); + private final List benchmarks = new ArrayList<>(); + public TranslationResult(TranslationManager translationManager) { this.translationManager = translationManager; } @@ -95,4 +101,32 @@ public Map getScratch() { public TranslationManager getTranslationManager() { return translationManager; } + + @Override + public void addBenchmark(@NotNull Benchmark b) { + this.benchmarks.add(b); + } + + public List getBenchmarks() { + return benchmarks; + } + + @Override + public void printBenchmark() { + StatisticsHolder.DefaultImpls.printBenchmark(this); + } + + @NotNull + @Override + public List getTranslatedFiles() { + return translationUnits.stream() + .map(TranslationUnitDeclaration::getName) + .collect(Collectors.toList()); + } + + @NotNull + @Override + public TranslationConfiguration getConfig() { + return translationManager.getConfig(); + } } diff --git a/cpg-core/src/main/java/de/fraunhofer/aisec/cpg/helpers/Benchmark.kt b/cpg-core/src/main/java/de/fraunhofer/aisec/cpg/helpers/Benchmark.kt index af0e42ff02..6132accfb1 100644 --- a/cpg-core/src/main/java/de/fraunhofer/aisec/cpg/helpers/Benchmark.kt +++ b/cpg-core/src/main/java/de/fraunhofer/aisec/cpg/helpers/Benchmark.kt @@ -25,15 +25,104 @@ */ package de.fraunhofer.aisec.cpg.helpers +import de.fraunhofer.aisec.cpg.TranslationConfiguration +import java.nio.file.Path import java.time.Duration import java.time.Instant +import java.util.* +import kotlin.IllegalArgumentException import org.slf4j.LoggerFactory +/** Interface definition to hold different statistics about the translation process. */ +interface StatisticsHolder { + val translatedFiles: List + val benchmarks: List + val config: TranslationConfiguration + + fun addBenchmark(b: Benchmark) + + /** Pretty-prints benchmark results for easy copying to GitHub issues. */ + fun printBenchmark() { + println("# Benchmark run ${UUID.randomUUID()}") + printMarkdown( + listOf( + listOf( + "Translation config", + "`${config.toString().replace(",", ", ").replace(":", ": ")}`" + ), + listOf("Number of files translated", translatedFiles.size), + listOf( + "Translated file(s)", + translatedFiles.map { + relativeOrAbsolute(Path.of(it), config.topLevel.toPath()) + } + ), + *benchmarks + .map { listOf("${it.caller}: ${it.message}", "${it.duration} ms") } + .toTypedArray(), + ), + listOf("Metric", "Value") + ) + } +} + +/** + * Prints a table of values and headers in markdown format. Table columns are automatically adjusted + * to the longest column. + */ +fun printMarkdown(table: List>, headers: List) { + val lengths = IntArray(headers.size) + + // first, we need to calculate the longest column per line + for (row in table) { + for (i in row.indices) { + val value = row[i].toString() + if (value.length > lengths[i]) { + lengths[i] = value.length + } + } + } + + // table header + val dash = lengths.joinToString(" | ", "| ", " |") { ("-".repeat(it)) } + var i = 0 + val header = headers.joinToString(" | ", "| ", " |") { it.padEnd(lengths[i++]) } + + println() + println(header) + println(dash) + + for (row in table) { + var rowIndex = 0 + val line = row.joinToString(" | ", "| ", " |") { it.toString().padEnd(lengths[rowIndex++]) } + println(line) + } + + println() +} + +/** + * This function will shorten / relativize the [path], if it is relative to [topLevel]. Otherwise, + * the full path will be returned. + */ +fun relativeOrAbsolute(path: Path, topLevel: Path): Path { + return try { + topLevel.toAbsolutePath().relativize(path) + } catch (ex: IllegalArgumentException) { + path + } +} + open class Benchmark @JvmOverloads -constructor(c: Class<*>, private val message: String, private var debug: Boolean = false) { +constructor( + c: Class<*>, + val message: String, + private var debug: Boolean = false, + private var holder: StatisticsHolder? = null +) { - private val caller: String + val caller: String private val start: Instant var duration: Long @@ -50,6 +139,9 @@ constructor(c: Class<*>, private val message: String, private var debug: Boolean log.info(msg) } + // update our holder, if we have any + holder?.addBenchmark(this) + return duration } diff --git a/cpg-core/src/testFixtures/java/de/fraunhofer/aisec/cpg/TestUtils.kt b/cpg-core/src/testFixtures/java/de/fraunhofer/aisec/cpg/TestUtils.kt index 2204450228..6a3439ad85 100644 --- a/cpg-core/src/testFixtures/java/de/fraunhofer/aisec/cpg/TestUtils.kt +++ b/cpg-core/src/testFixtures/java/de/fraunhofer/aisec/cpg/TestUtils.kt @@ -119,6 +119,28 @@ object TestUtils { usePasses: Boolean, configModifier: Consumer? = null ): List { + return analyzeWithResult(files, topLevel, usePasses, configModifier).translationUnits + } + + /** + * Default way of parsing a list of files into a full CPG. All default passes are applied + * + * @param topLevel The directory to traverse while looking for files to parse + * @param usePasses Whether the analysis should run passes after the initial phase + * @param configModifier An optional modifier for the config + * + * @return A list of [TranslationUnitDeclaration] nodes, representing the CPG roots + * @throws Exception Any exception thrown during the parsing process + */ + @JvmOverloads + @JvmStatic + @Throws(Exception::class) + fun analyzeWithResult( + files: List?, + topLevel: Path, + usePasses: Boolean, + configModifier: Consumer? = null + ): TranslationResult { val builder = TranslationConfiguration.builder() .sourceLocations(files) @@ -136,7 +158,7 @@ object TestUtils { configModifier?.accept(builder) val config = builder.build() val analyzer = TranslationManager.builder().config(config).build() - return analyzer.analyze().get().translationUnits + return analyzer.analyze().get() } /**