Log results of JUnit tests to a file

build.gradle

@@ -104,6 +104,11 @@ dependencies {
   graderImplementation group: 'commons-io', name: 'commons-io', version: '2.15.1'
   graderSources        group: 'commons-io', name: 'commons-io', version: '2.15.1', classifier: 'sources'
 
+  graderImplementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.15.2'
+  graderSources        group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.15.2', classifier: 'sources'
+  graderImplementation group: 'com.fasterxml.jackson.datatype', name: 'jackson-datatype-jsr310', version: '2.15.2'
+  graderImplementation group: 'com.fasterxml.jackson.datatype', name: 'jackson-datatype-jdk8', version: '2.15.2'
+
   // Swing stuff
   graderImplementation group: 'com.formdev', name: 'flatlaf', version: '3.3'
   graderImplementation group: 'com.formdev', name: 'flatlaf', version: '3.3', classifier: 'sources'

framework/tst/dslabs/framework/testing/junit/DSLabsTestCore.java

@@ -26,6 +26,7 @@
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Sets;
 import dslabs.framework.testing.utils.ClassSearch;
+import dslabs.framework.testing.utils.GlobalSettings;
 import java.util.Set;
 import org.apache.commons.cli.CommandLine;
 import org.apache.commons.cli.CommandLineParser;
@@ -68,7 +69,7 @@ static String fullTestNumber(Description d) {
     }
   }
 
-  private static void runRequest(Request request, RunListener listener) {
+  private static void runRequest(Request request, RunListener printer, RunListener... listeners) {
     final Runner runner = request.getRunner();
     final Result result = new Result();
     final RunNotifier notifier = new RunNotifier();
@@ -83,6 +84,10 @@ public void testFailure(Failure failure) {
             }
           });
     } else {
+      notifier.addListener(printer);
+    }
+
+    for (RunListener listener : listeners) {
       notifier.addListener(listener);
     }
 
@@ -270,7 +275,12 @@ public String describe() {
     // Sort methods and test classes
     request = request.sortWith(new TestOrder());
 
-    runRequest(request, new TestResultsPrinter());
+    if (GlobalSettings.testResultsOutputFile() != null) {
+      // For now, only attach a TestResultsLogger if we're actually logging to file.
+      runRequest(request, new TestResultsPrinter(), new TestResultsLogger());
+    } else {
+      runRequest(request, new TestResultsPrinter());
+    }
   }
 
   private DSLabsTestCore() {

framework/tst/dslabs/framework/testing/junit/TestResults.java

@@ -0,0 +1,95 @@
+/*
+ * Copyright (c) 2023 Ellis Michael
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package dslabs.framework.testing.junit;
+
+import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.PropertyAccessor;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.SerializationFeature;
+import com.fasterxml.jackson.databind.json.JsonMapper;
+import com.fasterxml.jackson.datatype.jdk8.Jdk8Module;
+import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
+import com.google.common.collect.ImmutableList;
+import java.io.File;
+import java.io.IOException;
+import java.io.Serializable;
+import java.time.Instant;
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Singular;
+import lombok.Value;
+import lombok.extern.jackson.Jacksonized;
+
+@Value
+@Builder
+@AllArgsConstructor(access = AccessLevel.PRIVATE)
+@Jacksonized
+@JsonIgnoreProperties
+class TestResults implements Serializable {
+  @Value
+  @Builder
+  @AllArgsConstructor(access = AccessLevel.PRIVATE)
+  @Jacksonized
+  static class TestResult implements Serializable {
+    String labName;
+    Integer part;
+    Integer testNumber;
+    String testDescription;
+    String testMethodName;
+    Integer pointsAvailable;
+    Integer pointsEarned;
+    @Singular ImmutableList<String> testCategories;
+
+    // TODO: use different System.out/System.err print streams in the
+    //  framework itself and log those here (to differentiate user/framework
+    //  logging).
+
+    String stdOutLog;
+    Boolean stdOutTruncated;
+    String stdErrLog;
+    Boolean stdErrTruncated;
+
+    Instant startTime;
+    Instant endTime;
+  }
+
+  @Singular ImmutableList<TestResult> results;
+
+  Instant startTime;
+  Instant endTime;
+
+  private static void configureObjectMapper(ObjectMapper mapper) {
+    mapper.enable(SerializationFeature.INDENT_OUTPUT);
+    mapper.registerModule(new Jdk8Module());
+    mapper.registerModule(new JavaTimeModule());
+    mapper.setVisibility(PropertyAccessor.FIELD, Visibility.ANY);
+  }
+
+  public void writeJsonToFile(String fileName) throws IOException {
+    JsonMapper mapper = new JsonMapper();
+    configureObjectMapper(mapper);
+    mapper.writeValue(new File(fileName), this);
+  }
+}

framework/tst/dslabs/framework/testing/junit/TestResultsLogger.java

@@ -0,0 +1,146 @@
+/*
+ * Copyright (c) 2023 Ellis Michael ([email protected])
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package dslabs.framework.testing.junit;
+
+import com.google.common.collect.ImmutableList;
+import dslabs.framework.testing.junit.TestResults.TestResult;
+import dslabs.framework.testing.junit.TestResults.TestResult.TestResultBuilder;
+import dslabs.framework.testing.junit.TestResults.TestResultsBuilder;
+import dslabs.framework.testing.utils.GlobalSettings;
+import dslabs.framework.testing.utils.TeeStdOutErr;
+import java.io.IOException;
+import java.time.Instant;
+import java.util.Arrays;
+import org.junit.experimental.categories.Category;
+import org.junit.runner.Description;
+import org.junit.runner.Result;
+import org.junit.runner.notification.Failure;
+import org.junit.runner.notification.RunListener;
+
+/** Records results of test runs and, if enabled, logs them to a file. */
+class TestResultsLogger extends RunListener {
+  private static boolean isListening = false;
+
+  private final TestResultsBuilder testResultsBuilder = TestResults.builder();
+  private TestResultBuilder testResultBuilder = null;
+
+  private Description testDescription;
+
+  private static int testNumber(Description d) {
+    assert d.isTest();
+    String n = d.getMethodName();
+    return Integer.parseInt(n.replaceFirst("test(\\d+)\\w+", "$1"));
+  }
+
+  @Override
+  public void testRunStarted(Description description) throws Exception {
+    // Can only run one instance of the listener at a time
+    assert !isListening;
+    isListening = true;
+    testResultsBuilder.startTime(Instant.now());
+  }
+
+  @Override
+  public void testRunFinished(Result result) throws IOException {
+    testResultsBuilder.endTime(Instant.now());
+    TestResults results = testResultsBuilder.build();
+
+    // Write out the results to a file if enabled
+    if (GlobalSettings.testResultsOutputFile() != null) {
+      results.writeJsonToFile(GlobalSettings.testResultsOutputFile());
+    }
+
+    assert isListening;
+    isListening = false;
+  }
+
+  @Override
+  public void testStarted(Description description) throws NoSuchMethodException {
+    testDescription = description;
+
+    testResultBuilder = TestResult.builder();
+    testResultBuilder.startTime(Instant.now());
+
+    Class<?> testClass = description.getTestClass();
+
+    Lab labName = testClass.getAnnotation(Lab.class);
+    if (labName != null) {
+      testResultBuilder.labName(labName.value());
+    }
+
+    Part part = testClass.getAnnotation(Part.class);
+    if (part != null) {
+      testResultBuilder.part(part.value());
+    }
+
+    testResultBuilder.testNumber(testNumber(description));
+
+    TestDescription testDescription = description.getAnnotation(TestDescription.class);
+    if (testDescription != null) {
+      testResultBuilder.testDescription(testDescription.value());
+    }
+
+    TestPointValue testPointValue = description.getAnnotation(TestPointValue.class);
+    if (testPointValue != null) {
+      testResultBuilder.pointsAvailable(testPointValue.value());
+      testResultBuilder.pointsEarned(testPointValue.value());
+    }
+
+    testResultBuilder.testMethodName(description.getMethodName());
+
+    Category categories = description.getAnnotation(Category.class);
+    if (categories != null) {
+      testResultBuilder.testCategories(
+          Arrays.stream(categories.value())
+              .map(Class::getName)
+              .collect(ImmutableList.toImmutableList()));
+    }
+
+    // Install tees for System.out and System.err.
+    TeeStdOutErr.installTees();
+  }
+
+  @Override
+  public void testFailure(Failure failure) {
+    assert testDescription != null;
+    if (testDescription.getAnnotation(TestPointValue.class) != null) {
+      testResultBuilder.pointsEarned(0);
+    }
+  }
+
+  @Override
+  public void testFinished(Description description) {
+    // Log the test result
+    testResultBuilder.endTime(Instant.now());
+
+    var tee = TeeStdOutErr.clearTees();
+    testResultBuilder.stdOutLog(tee.stdOut());
+    testResultBuilder.stdOutTruncated(tee.stdOutTruncated());
+    testResultBuilder.stdErrLog(tee.stdErr());
+    testResultBuilder.stdErrTruncated(tee.stdErrTruncated());
+
+    testResultsBuilder.result(testResultBuilder.build());
+    testResultBuilder = null;
+    testDescription = null;
+  }
+}

framework/tst/dslabs/framework/testing/utils/GlobalSettings.java

@@ -30,6 +30,7 @@
 import java.util.logging.Level;
 import java.util.logging.LogManager;
 import java.util.logging.Logger;
+import javax.annotation.Nullable;
 import lombok.Getter;
 import lombok.Setter;
 
@@ -62,6 +63,19 @@ public final class GlobalSettings {
   private static final boolean timeoutsDisabled =
       Boolean.parseBoolean(lookupWithDefault("testTimeoutsDisabled", "false"));
 
+  /** The file to print test results to (in JSON format). */
+  @Getter @Nullable
+  private static final String testResultsOutputFile = lookupWithDefault("resultsOutputFile", null);
+
+  /**
+   * The maximum log size (in bytes) the framework should retain and print to file. This size limit
+   * applies to each test independently and applies to stdout/stderr independently. If this is 0,
+   * then logging is effectively disabled. If this is less than 0, then there is no limit (default).
+   */
+  @Getter
+  private static final int maximumStdOutErrLogSize =
+      Integer.parseInt(lookupWithDefault("maxLogSize", "-1"));
+
   static {
     System.setProperty(
         "java.util.logging.SimpleFormatter.format", "[%4$-7s] [%1$tF %1$tT] [%3$s] %5$s%6$s%n");