Merge commit '8de652368cd4accf7d0f6801ce7dcb8d75744c25' into feature/…

…directly-load-acm-files
ArDoCo · Aug 23, 2024 · 6efe1d2 · 6efe1d2
2 parents e273c6a + 8de6523
commit 6efe1d2
Show file tree

Hide file tree

Showing 32 changed files with 472 additions and 1,987 deletions.
diff --git a/tests/integration-tests/tests-base/pom.xml b/tests/integration-tests/tests-base/pom.xml
@@ -17,6 +17,12 @@
       <groupId>com.tngtech.archunit</groupId>
       <artifactId>archunit-junit5</artifactId>
     </dependency>
+    <dependency>
+      <groupId>io.github.ardoco</groupId>
+      <artifactId>metrics</artifactId>
+      <version>0.1.1-SNAPSHOT</version>
+      <scope>compile</scope>
+    </dependency>
     <dependency>
       <groupId>io.github.ardoco.core</groupId>
       <artifactId>common</artifactId>

diff --git a/...ration-tests/tests-base/src/main/java/edu/kit/kastel/mcse/ardoco/core/tests/TestUtil.java b/...ration-tests/tests-base/src/main/java/edu/kit/kastel/mcse/ardoco/core/tests/TestUtil.java
@@ -3,17 +3,14 @@
 
 import java.util.Locale;
 import java.util.Set;
-import java.util.stream.Collectors;
 
 import org.eclipse.collections.api.collection.ImmutableCollection;
-import org.eclipse.collections.api.factory.Lists;
-import org.eclipse.collections.api.list.ImmutableList;
 import org.slf4j.Logger;
 
 import edu.kit.kastel.mcse.ardoco.core.api.output.ArDoCoResult;
 import edu.kit.kastel.mcse.ardoco.core.tests.eval.results.EvaluationResults;
 import edu.kit.kastel.mcse.ardoco.core.tests.eval.results.ExpectedResults;
-import edu.kit.kastel.mcse.ardoco.core.tests.eval.results.ResultMatrix;
+import edu.kit.kastel.mcse.ardoco.metrics.ClassificationMetricsCalculator;
 
 /**
  * This utility class provides methods for running the tests, especially regarding the evaluations.
@@ -37,20 +34,18 @@ public static <T> EvaluationResults<T> compareTLR(ArDoCoResult arDoCoResult, Imm
         Set<T> distinctTraceLinks = new java.util.LinkedHashSet<>(results.castToCollection());
         Set<T> distinctGoldStandard = new java.util.LinkedHashSet<>(goldStandard.castToCollection());
 
-        // True Positives are the trace links that are contained on both lists
-        Set<T> truePositives = distinctTraceLinks.stream().filter(distinctGoldStandard::contains).collect(Collectors.toSet());
-        ImmutableList<T> truePositivesList = Lists.immutable.ofAll(truePositives);
+        int sentences = arDoCoResult.getText().getSentences().size();
+        int modelElements = 0;
+        for (var model : arDoCoResult.getModelIds()) {
+            modelElements += arDoCoResult.getModelState(model).getInstances().size();
+        }
 
-        // False Positives are the trace links that are only contained in the result set
-        Set<T> falsePositives = distinctTraceLinks.stream().filter(tl -> !distinctGoldStandard.contains(tl)).collect(Collectors.toSet());
-        ImmutableList<T> falsePositivesList = Lists.immutable.ofAll(falsePositives);
+        int confusionMatrixSum = sentences * modelElements;
 
-        // False Negatives are the trace links that are only contained in the gold standard
-        Set<T> falseNegatives = distinctGoldStandard.stream().filter(tl -> !distinctTraceLinks.contains(tl)).collect(Collectors.toSet());
-        ImmutableList<T> falseNegativesList = Lists.immutable.ofAll(falseNegatives);
+        var calculator = ClassificationMetricsCalculator.getInstance();
 
-        int trueNegatives = TestUtil.calculateTrueNegativesForTLR(arDoCoResult, truePositives.size(), falsePositives.size(), falseNegatives.size());
-        return EvaluationResults.createEvaluationResults(new ResultMatrix<>(truePositivesList, trueNegatives, falsePositivesList, falseNegativesList));
+        var classification = calculator.calculateMetrics(distinctTraceLinks, distinctGoldStandard, confusionMatrixSum);
+        return new EvaluationResults<>(classification);
     }
 
     /**
@@ -67,57 +62,11 @@ public static <T> EvaluationResults<T> compareInconsistencies(ArDoCoResult arDoC
         Set<T> distinctTraceLinks = new java.util.LinkedHashSet<>(results.castToCollection());
         Set<T> distinctGoldStandard = new java.util.LinkedHashSet<>(goldStandard.castToCollection());
 
-        // True Positives are the trace links that are contained on both lists
-        Set<T> truePositives = distinctTraceLinks.stream().filter(distinctGoldStandard::contains).collect(Collectors.toSet());
-        ImmutableList<T> truePositivesList = Lists.immutable.ofAll(truePositives);
-
-        // False Positives are the trace links that are only contained in the result set
-        Set<T> falsePositives = distinctTraceLinks.stream().filter(tl -> !distinctGoldStandard.contains(tl)).collect(Collectors.toSet());
-        ImmutableList<T> falsePositivesList = Lists.immutable.ofAll(falsePositives);
-
-        // False Negatives are the trace links that are only contained in the gold standard
-        Set<T> falseNegatives = distinctGoldStandard.stream().filter(tl -> !distinctTraceLinks.contains(tl)).collect(Collectors.toSet());
-        ImmutableList<T> falseNegativesList = Lists.immutable.ofAll(falseNegatives);
-
-        int trueNegatives = TestUtil.calculateTrueNegativesForInconsistencies(arDoCoResult, truePositives.size(), falsePositives.size(), falseNegatives.size());
-        return EvaluationResults.createEvaluationResults(new ResultMatrix<>(truePositivesList, trueNegatives, falsePositivesList, falseNegativesList));
-    }
-
-    /**
-     * Calculates the number of true negatives based on the given {@link ArDoCoResult} and the calculated {@link EvaluationResults evaluation results}. Uses the
-     * total sum of all entries in the confusion matrix and then substracts the true positives, false positives, and false negatives.
-     *
-     * @param arDoCoResult   the output of ArDoCo
-     * @param truePositives  nr of true positives
-     * @param falsePositives nr of false positives
-     * @param falseNegatives nr of false negatives
-     * @return the number of true negatives
-     */
-    public static int calculateTrueNegativesForTLR(ArDoCoResult arDoCoResult, int truePositives, int falsePositives, int falseNegatives) {
-        int sentences = arDoCoResult.getText().getSentences().size();
-        int modelElements = 0;
-        for (var model : arDoCoResult.getModelIds()) {
-            modelElements += arDoCoResult.getModelState(model).getInstances().size();
-        }
-
-        int confusionMatrixSum = sentences * modelElements;
-        return confusionMatrixSum - (truePositives + falsePositives + falseNegatives);
-    }
-
-    /**
-     * Calculates the number of true negatives based on the given {@link ArDoCoResult} and the calculated {@link EvaluationResults evaluation results}. Uses the
-     * total sum of all sentences in the {@link ArDoCoResult} and then substracts the true positives, false positives, and false negatives.
-     *
-     * @param arDoCoResult   the output of ArDoCo
-     * @param truePositives  nr of true positives
-     * @param falsePositives nr of false positives
-     * @param falseNegatives nr of false negatives
-     * @return the number of true negatives
-     */
-    public static int calculateTrueNegativesForInconsistencies(ArDoCoResult arDoCoResult, int truePositives, int falsePositives, int falseNegatives) {
-        int numberOfSentences = arDoCoResult.getText().getSentences().size();
-        return numberOfSentences - (truePositives + falsePositives + falseNegatives);
+        int confusionMatrixSum = arDoCoResult.getText().getSentences().size();
 
+        var calculator = ClassificationMetricsCalculator.getInstance();
+        var classification = calculator.calculateMetrics(distinctTraceLinks, distinctGoldStandard, confusionMatrixSum);
+        return new EvaluationResults<>(classification);
     }
 
     /**
@@ -168,19 +117,6 @@ public static void logExplicitResults(Logger logger, String name, EvaluationResu
         logger.info(logString);
     }
 
-    /**
-     * Log the provided {@link EvaluationResults} using the provided logger and name. Additionally, provided the expected results.
-     *
-     * @param logger          Logger to use
-     * @param name            Name to show in the output
-     * @param results         the results
-     * @param expectedResults the expected results
-     */
-    public static void logResultsWithExpected(Logger logger, String name, EvaluationResults<?> results, ExpectedResults expectedResults) {
-        var infoString = String.format(Locale.ENGLISH, "%n%s:%n%s", name, results.getResultStringWithExpected(expectedResults));
-        logger.info(infoString);
-    }
-
     public static void logExtendedResultsWithExpected(Logger logger, Object testClass, String name, EvaluationResults<?> results,
             ExpectedResults expectedResults) {
         var infoString = String.format(Locale.ENGLISH, """

diff --git a/...main/java/edu/kit/kastel/mcse/ardoco/core/tests/architecture/DeterministicArDoCoTest.java b/...main/java/edu/kit/kastel/mcse/ardoco/core/tests/architecture/DeterministicArDoCoTest.java
@@ -50,7 +50,7 @@ public class DeterministicArDoCoTest {
 
     @ArchTest
     public static final ArchRule forbidUnorderedSetsAndMaps = noClasses().that()
-            .resideOutsideOfPackages("..tests..")
+            .resideOutsideOfPackages("..tests..", "..metrics..")
             .and(areNotDirectlyAnnotatedWith(Deterministic.class))
             .should()
             .accessClassesThat(areForbiddenClasses())

diff --git a/...ests/tests-base/src/main/java/edu/kit/kastel/mcse/ardoco/core/tests/eval/CodeProject.java b/...ests/tests-base/src/main/java/edu/kit/kastel/mcse/ardoco/core/tests/eval/CodeProject.java
@@ -1,10 +1,10 @@
 /* Licensed under MIT 2023-2024. */
 package edu.kit.kastel.mcse.ardoco.core.tests.eval;
 
+import static edu.kit.kastel.mcse.ardoco.core.tests.eval.ProjectHelper.loadFileFromResources;
+
 import java.io.File;
-import java.io.FileOutputStream;
 import java.io.IOException;
-import java.io.InputStream;
 import java.nio.file.Files;
 import java.nio.file.Paths;
 import java.util.List;
@@ -145,41 +145,18 @@ public String getCommitHash() {
     }
 
     /**
-     * {@return path of the code directory}
-     */
-    public String getCodeLocation() {
-        return getTemporaryCodeLocation().getAbsolutePath();
-    }
-
-    /**
-     * {@return the directory of the code model}
-     */
-    public String getCodeModelDirectory() {
-        try {
-            loadCodeModelFromResourcesIfNeeded();
-            return getTemporaryCodeLocation().getAbsolutePath();
-        } catch (IOException e) {
-            logger.error(e.getMessage(), e);
-            return null;
-        }
-    }
-
-    /**
-     * Loads the code from resources or from the code directoy cache
+     * Get Code Location (ACM File or Temporary Directory)
+     * 
+     * @param acmFile If true, the ACM file is loaded from resources
      *
-     * @throws IOException Can occur during file operations
      */
-    public void loadCodeModelFromResourcesIfNeeded() throws IOException {
-        if (ProjectHelper.ANALYZE_CODE_DIRECTLY.get())
-            return;
-
-        File temporaryCodeLocation = getTemporaryCodeLocation();
-        File codeModelFile = new File(temporaryCodeLocation + "/codeModel.acm");
-        try (InputStream is = getClass().getResourceAsStream(this.codeModelLocationInResources)) {
-            try (FileOutputStream fos = new FileOutputStream(codeModelFile)) {
-                is.transferTo(fos);
-            }
+    public File getCodeLocation(boolean acmFile) {
+        if (acmFile) {
+            // If ACM load file from resources
+            return loadFileFromResources(this.codeModelLocationInResources);
         }
+
+        return getTemporaryCodeLocation();
     }
 
     /**
@@ -202,7 +179,7 @@ public ExpectedResults getExpectedResultsForSadSamCode() {
      * @see TraceLinkUtilities#createTraceLinkString(String, String)
      */
     public ImmutableList<String> getSamCodeGoldStandard() {
-        File samCodeGoldStandardFile = ProjectHelper.loadFileFromResources(samCodeGoldStandardLocation);
+        File samCodeGoldStandardFile = loadFileFromResources(samCodeGoldStandardLocation);
         List<String> lines = getLinesFromGoldStandardFile(samCodeGoldStandardFile);
 
         MutableList<String> goldStandard = Lists.mutable.empty();
@@ -222,7 +199,7 @@ public ImmutableList<String> getSamCodeGoldStandard() {
      * {@return all lines from the gold standard in csv format}
      */
     public ImmutableList<String> getSadCodeGoldStandard() {
-        File sadCodeGoldStandardFile = ProjectHelper.loadFileFromResources(sadCodeGoldStandardLocation);
+        File sadCodeGoldStandardFile = loadFileFromResources(sadCodeGoldStandardLocation);
         List<String> lines = getLinesFromGoldStandardFile(sadCodeGoldStandardFile);
         return Lists.immutable.ofAll(lines);
     }