Skip to content

Commit

Permalink
Squashed 'tests/integration-tests/' changes from fb741c9..2d2daa0
Browse files Browse the repository at this point in the history 
2d2daa0 Merge branch 'main' into feature/directly-load-acm-files
8f33b88 Update to Metrics 0.1.1-SNAPSHOT
319ff9f Update name in MESLink
7b0e7cd Loading of ACM files is possible for code now.
3fc68e7 Update to Metrics to 0.1.0
e4480d0 Extract Historic Project. Remove TLR Files. Cleanup EvaluationIT
a77bdcf Cleanup TLRHelper
7ac3d95 Fix Name of Link
be5345e Remove metrics
9b5c740 Remove aggregated averages for TLR / ID
ae8fe30 Update Expected Results
e227637 Merge branch 'main' into feature/metrics
237d987 Use MicroAverage to aggregate MMEs per ptoject
26cf99a Make it compile
2b2fca4 Work on using metrics project

git-subtree-dir: tests/integration-tests
git-subtree-split: 2d2daa076c383fcb3eaf6611cc0af5eb48182a84
  • Loading branch information
@dfuchss
dfuchss committed Aug 23, 2024
1 parent 8c440bc commit 8de6523
Show file tree
Hide file tree
Showing 32 changed files with 472 additions and 1,987 deletions.
6 changes: 6 additions & 0 deletions tests-base/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,12 @@
<groupId>com.tngtech.archunit</groupId>
<artifactId>archunit-junit5</artifactId>
</dependency>
<dependency>
<groupId>io.github.ardoco</groupId>
<artifactId>metrics</artifactId>
<version>0.1.1-SNAPSHOT</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>io.github.ardoco.core</groupId>
<artifactId>common</artifactId>
Expand Down
92 changes: 14 additions & 78 deletions tests-base/src/main/java/edu/kit/kastel/mcse/ardoco/core/tests/TestUtil.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,17 +3,14 @@

import java.util.Locale;
import java.util.Set;
import java.util.stream.Collectors;

import org.eclipse.collections.api.collection.ImmutableCollection;
import org.eclipse.collections.api.factory.Lists;
import org.eclipse.collections.api.list.ImmutableList;
import org.slf4j.Logger;

import edu.kit.kastel.mcse.ardoco.core.api.output.ArDoCoResult;
import edu.kit.kastel.mcse.ardoco.core.tests.eval.results.EvaluationResults;
import edu.kit.kastel.mcse.ardoco.core.tests.eval.results.ExpectedResults;
import edu.kit.kastel.mcse.ardoco.core.tests.eval.results.ResultMatrix;
import edu.kit.kastel.mcse.ardoco.metrics.ClassificationMetricsCalculator;

/**
* This utility class provides methods for running the tests, especially regarding the evaluations.
Expand All @@ -37,20 +34,18 @@ public static <T> EvaluationResults<T> compareTLR(ArDoCoResult arDoCoResult, Imm
Set<T> distinctTraceLinks = new java.util.LinkedHashSet<>(results.castToCollection());
Set<T> distinctGoldStandard = new java.util.LinkedHashSet<>(goldStandard.castToCollection());

// True Positives are the trace links that are contained on both lists
Set<T> truePositives = distinctTraceLinks.stream().filter(distinctGoldStandard::contains).collect(Collectors.toSet());
ImmutableList<T> truePositivesList = Lists.immutable.ofAll(truePositives);
int sentences = arDoCoResult.getText().getSentences().size();
int modelElements = 0;
for (var model : arDoCoResult.getModelIds()) {
modelElements += arDoCoResult.getModelState(model).getInstances().size();
}

// False Positives are the trace links that are only contained in the result set
Set<T> falsePositives = distinctTraceLinks.stream().filter(tl -> !distinctGoldStandard.contains(tl)).collect(Collectors.toSet());
ImmutableList<T> falsePositivesList = Lists.immutable.ofAll(falsePositives);
int confusionMatrixSum = sentences * modelElements;

// False Negatives are the trace links that are only contained in the gold standard
Set<T> falseNegatives = distinctGoldStandard.stream().filter(tl -> !distinctTraceLinks.contains(tl)).collect(Collectors.toSet());
ImmutableList<T> falseNegativesList = Lists.immutable.ofAll(falseNegatives);
var calculator = ClassificationMetricsCalculator.getInstance();

int trueNegatives = TestUtil.calculateTrueNegativesForTLR(arDoCoResult, truePositives.size(), falsePositives.size(), falseNegatives.size());
return EvaluationResults.createEvaluationResults(new ResultMatrix<>(truePositivesList, trueNegatives, falsePositivesList, falseNegativesList));
var classification = calculator.calculateMetrics(distinctTraceLinks, distinctGoldStandard, confusionMatrixSum);
return new EvaluationResults<>(classification);
}

/**
Expand All @@ -67,57 +62,11 @@ public static <T> EvaluationResults<T> compareInconsistencies(ArDoCoResult arDoC
Set<T> distinctTraceLinks = new java.util.LinkedHashSet<>(results.castToCollection());
Set<T> distinctGoldStandard = new java.util.LinkedHashSet<>(goldStandard.castToCollection());

// True Positives are the trace links that are contained on both lists
Set<T> truePositives = distinctTraceLinks.stream().filter(distinctGoldStandard::contains).collect(Collectors.toSet());
ImmutableList<T> truePositivesList = Lists.immutable.ofAll(truePositives);

// False Positives are the trace links that are only contained in the result set
Set<T> falsePositives = distinctTraceLinks.stream().filter(tl -> !distinctGoldStandard.contains(tl)).collect(Collectors.toSet());
ImmutableList<T> falsePositivesList = Lists.immutable.ofAll(falsePositives);

// False Negatives are the trace links that are only contained in the gold standard
Set<T> falseNegatives = distinctGoldStandard.stream().filter(tl -> !distinctTraceLinks.contains(tl)).collect(Collectors.toSet());
ImmutableList<T> falseNegativesList = Lists.immutable.ofAll(falseNegatives);

int trueNegatives = TestUtil.calculateTrueNegativesForInconsistencies(arDoCoResult, truePositives.size(), falsePositives.size(), falseNegatives.size());
return EvaluationResults.createEvaluationResults(new ResultMatrix<>(truePositivesList, trueNegatives, falsePositivesList, falseNegativesList));
}

/**
* Calculates the number of true negatives based on the given {@link ArDoCoResult} and the calculated {@link EvaluationResults evaluation results}. Uses the
* total sum of all entries in the confusion matrix and then substracts the true positives, false positives, and false negatives.
*
* @param arDoCoResult the output of ArDoCo
* @param truePositives nr of true positives
* @param falsePositives nr of false positives
* @param falseNegatives nr of false negatives
* @return the number of true negatives
*/
public static int calculateTrueNegativesForTLR(ArDoCoResult arDoCoResult, int truePositives, int falsePositives, int falseNegatives) {
int sentences = arDoCoResult.getText().getSentences().size();
int modelElements = 0;
for (var model : arDoCoResult.getModelIds()) {
modelElements += arDoCoResult.getModelState(model).getInstances().size();
}

int confusionMatrixSum = sentences * modelElements;
return confusionMatrixSum - (truePositives + falsePositives + falseNegatives);
}

/**
* Calculates the number of true negatives based on the given {@link ArDoCoResult} and the calculated {@link EvaluationResults evaluation results}. Uses the
* total sum of all sentences in the {@link ArDoCoResult} and then substracts the true positives, false positives, and false negatives.
*
* @param arDoCoResult the output of ArDoCo
* @param truePositives nr of true positives
* @param falsePositives nr of false positives
* @param falseNegatives nr of false negatives
* @return the number of true negatives
*/
public static int calculateTrueNegativesForInconsistencies(ArDoCoResult arDoCoResult, int truePositives, int falsePositives, int falseNegatives) {
int numberOfSentences = arDoCoResult.getText().getSentences().size();
return numberOfSentences - (truePositives + falsePositives + falseNegatives);
int confusionMatrixSum = arDoCoResult.getText().getSentences().size();

var calculator = ClassificationMetricsCalculator.getInstance();
var classification = calculator.calculateMetrics(distinctTraceLinks, distinctGoldStandard, confusionMatrixSum);
return new EvaluationResults<>(classification);
}

/**
Expand Down Expand Up @@ -168,19 +117,6 @@ public static void logExplicitResults(Logger logger, String name, EvaluationResu
logger.info(logString);
}

/**
* Log the provided {@link EvaluationResults} using the provided logger and name. Additionally, provided the expected results.
*
* @param logger Logger to use
* @param name Name to show in the output
* @param results the results
* @param expectedResults the expected results
*/
public static void logResultsWithExpected(Logger logger, String name, EvaluationResults<?> results, ExpectedResults expectedResults) {
var infoString = String.format(Locale.ENGLISH, "%n%s:%n%s", name, results.getResultStringWithExpected(expectedResults));
logger.info(infoString);
}

public static void logExtendedResultsWithExpected(Logger logger, Object testClass, String name, EvaluationResults<?> results,
ExpectedResults expectedResults) {
var infoString = String.format(Locale.ENGLISH, """
Expand Down
2 changes: 1 addition & 1 deletion ...main/java/edu/kit/kastel/mcse/ardoco/core/tests/architecture/DeterministicArDoCoTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ public class DeterministicArDoCoTest {

@ArchTest
public static final ArchRule forbidUnorderedSetsAndMaps = noClasses().that()
.resideOutsideOfPackages("..tests..")
.resideOutsideOfPackages("..tests..", "..metrics..")
.and(areNotDirectlyAnnotatedWith(Deterministic.class))
.should()
.accessClassesThat(areForbiddenClasses())
Expand Down
49 changes: 13 additions & 36 deletions tests-base/src/main/java/edu/kit/kastel/mcse/ardoco/core/tests/eval/CodeProject.java
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
/* Licensed under MIT 2023-2024. */
package edu.kit.kastel.mcse.ardoco.core.tests.eval;

import static edu.kit.kastel.mcse.ardoco.core.tests.eval.ProjectHelper.loadFileFromResources;

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.List;
Expand Down Expand Up @@ -145,41 +145,18 @@ public String getCommitHash() {
}

/**
* {@return path of the code directory}
*/
public String getCodeLocation() {
return getTemporaryCodeLocation().getAbsolutePath();
}

/**
* {@return the directory of the code model}
*/
public String getCodeModelDirectory() {
try {
loadCodeModelFromResourcesIfNeeded();
return getTemporaryCodeLocation().getAbsolutePath();
} catch (IOException e) {
logger.error(e.getMessage(), e);
return null;
}
}

/**
* Loads the code from resources or from the code directoy cache
* Get Code Location (ACM File or Temporary Directory)
*
* @param acmFile If true, the ACM file is loaded from resources
*
* @throws IOException Can occur during file operations
*/
public void loadCodeModelFromResourcesIfNeeded() throws IOException {
if (ProjectHelper.ANALYZE_CODE_DIRECTLY.get())
return;

File temporaryCodeLocation = getTemporaryCodeLocation();
File codeModelFile = new File(temporaryCodeLocation + "/codeModel.acm");
try (InputStream is = getClass().getResourceAsStream(this.codeModelLocationInResources)) {
try (FileOutputStream fos = new FileOutputStream(codeModelFile)) {
is.transferTo(fos);
}
public File getCodeLocation(boolean acmFile) {
if (acmFile) {
// If ACM load file from resources
return loadFileFromResources(this.codeModelLocationInResources);
}

return getTemporaryCodeLocation();
}

/**
Expand All @@ -202,7 +179,7 @@ public ExpectedResults getExpectedResultsForSadSamCode() {
* @see TraceLinkUtilities#createTraceLinkString(String, String)
*/
public ImmutableList<String> getSamCodeGoldStandard() {
File samCodeGoldStandardFile = ProjectHelper.loadFileFromResources(samCodeGoldStandardLocation);
File samCodeGoldStandardFile = loadFileFromResources(samCodeGoldStandardLocation);
List<String> lines = getLinesFromGoldStandardFile(samCodeGoldStandardFile);

MutableList<String> goldStandard = Lists.mutable.empty();
Expand All @@ -222,7 +199,7 @@ public ImmutableList<String> getSamCodeGoldStandard() {
* {@return all lines from the gold standard in csv format}
*/
public ImmutableList<String> getSadCodeGoldStandard() {
File sadCodeGoldStandardFile = ProjectHelper.loadFileFromResources(sadCodeGoldStandardLocation);
File sadCodeGoldStandardFile = loadFileFromResources(sadCodeGoldStandardLocation);
List<String> lines = getLinesFromGoldStandardFile(sadCodeGoldStandardFile);
return Lists.immutable.ofAll(lines);
}
Expand Down
Loading

0 comments on commit 8de6523

Please sign in to comment.