diff --git a/src/main/java/hudson/tasks/junit/JUnitParser.java b/src/main/java/hudson/tasks/junit/JUnitParser.java index 8aaa2e181..1a0b9b93e 100644 --- a/src/main/java/hudson/tasks/junit/JUnitParser.java +++ b/src/main/java/hudson/tasks/junit/JUnitParser.java @@ -51,13 +51,16 @@ public class JUnitParser extends TestResultParser { private final boolean keepLongStdio; private final boolean allowEmptyResults; + private final boolean makeUnstable; + /** Generally unused, but present for extension compatibility. */ @Deprecated public JUnitParser() { - this(false, false); + this(false, false, true); } - + + /** * @param keepLongStdio if true, retain a suite's complete stdout/stderr even if this is huge and the suite passed * @since 1.358 @@ -66,8 +69,10 @@ public JUnitParser() { public JUnitParser(boolean keepLongStdio) { this.keepLongStdio = keepLongStdio; this.allowEmptyResults = false; + this.makeUnstable = true; } - + + /** * @param keepLongStdio if true, retain a suite's complete stdout/stderr even if this is huge and the suite passed * @param allowEmptyResults if true, empty results are allowed @@ -76,6 +81,19 @@ public JUnitParser(boolean keepLongStdio) { public JUnitParser(boolean keepLongStdio, boolean allowEmptyResults) { this.keepLongStdio = keepLongStdio; this.allowEmptyResults = allowEmptyResults; + this.makeUnstable = true; + } + + /** + * @param keepLongStdio if true, retain a suite's complete stdout/stderr even if this is huge and the suite passed + * @param allowEmptyResults if true, empty results are allowed + * @param makeUnstable if true, build will be Unstable if there are any failed test cases + * @since 1.10 + */ + public JUnitParser(boolean keepLongStdio, boolean allowEmptyResults, boolean makeUnstable) { + this.keepLongStdio = keepLongStdio; + this.allowEmptyResults = allowEmptyResults; + this.makeUnstable = makeUnstable; } @Override @@ -112,7 +130,7 @@ public TestResult parseResult(String testResultLocations, Run build, Pipeli // also get code that deals with testDataPublishers from JUnitResultArchiver.perform return workspace.act(new ParseResultCallable(testResultLocations, buildTime, timeOnMaster, keepLongStdio, - allowEmptyResults, pipelineTestDetails)); + allowEmptyResults, makeUnstable, pipelineTestDetails)); } private static final class ParseResultCallable extends MasterToSlaveFileCallable { @@ -121,16 +139,19 @@ private static final class ParseResultCallable extends MasterToSlaveFileCallable private final long nowMaster; private final boolean keepLongStdio; private final boolean allowEmptyResults; + private final boolean makeUnstable; private final PipelineTestDetails pipelineTestDetails; private ParseResultCallable(String testResults, long buildTime, long nowMaster, boolean keepLongStdio, boolean allowEmptyResults, + boolean makeUnstable, PipelineTestDetails pipelineTestDetails) { this.buildTime = buildTime; this.testResults = testResults; this.nowMaster = nowMaster; this.keepLongStdio = keepLongStdio; this.allowEmptyResults = allowEmptyResults; + this.makeUnstable = makeUnstable; this.pipelineTestDetails = pipelineTestDetails; } diff --git a/src/main/java/hudson/tasks/junit/JUnitResultArchiver.java b/src/main/java/hudson/tasks/junit/JUnitResultArchiver.java index febbfc84f..963d0b956 100644 --- a/src/main/java/hudson/tasks/junit/JUnitResultArchiver.java +++ b/src/main/java/hudson/tasks/junit/JUnitResultArchiver.java @@ -89,10 +89,12 @@ public class JUnitResultArchiver extends Recorder implements SimpleBuildStep, JU * If true, don't throw exception on missing test results or no files found. */ private boolean allowEmptyResults; + private boolean makeUnstable; @DataBoundConstructor public JUnitResultArchiver(String testResults) { this.testResults = testResults; + this.makeUnstable = true; } @Deprecated @@ -107,6 +109,7 @@ public JUnitResultArchiver( boolean keepLongStdio, DescribableList> testDataPublishers) { this(testResults, keepLongStdio, testDataPublishers, 1.0); + this.makeUnstable = true; } @Deprecated @@ -121,6 +124,22 @@ public JUnitResultArchiver( setHealthScaleFactor(healthScaleFactor); setAllowEmptyResults(false); } + + + @Deprecated + public JUnitResultArchiver( + String testResults, + boolean keepLongStdio, + DescribableList> testDataPublishers, + double healthScaleFactor, + boolean makeUnstable) { + this.testResults = testResults; + setKeepLongStdio(keepLongStdio); + setTestDataPublishers(testDataPublishers == null ? Collections.emptyList() : testDataPublishers); + setHealthScaleFactor(healthScaleFactor); + setAllowEmptyResults(false); + setMakeUnstable(true); + } private TestResult parse(String expandedTestResults, Run run, @Nonnull FilePath workspace, Launcher launcher, TaskListener listener) throws IOException, InterruptedException @@ -133,7 +152,7 @@ private static TestResult parse(@Nonnull JUnitTask task, PipelineTestDetails pip String expandedTestResults, Run run, @Nonnull FilePath workspace, Launcher launcher, TaskListener listener) throws IOException, InterruptedException { - return new JUnitParser(task.isKeepLongStdio(), task.isAllowEmptyResults()) + return new JUnitParser(task.isKeepLongStdio(), task.isAllowEmptyResults(), task.isMakeUnstable()) .parseResult(expandedTestResults, run, pipelineTestDetails, workspace, launcher, listener); } @@ -153,8 +172,9 @@ public void perform(Run build, FilePath workspace, Launcher launcher, TaskListener listener) throws InterruptedException, IOException { TestResultAction action = parseAndAttach(this, null, build, workspace, launcher, listener); - if (action != null && action.getResult().getFailCount() > 0) + if ((action != null) && (action.getResult().getFailCount() > 0)) build.setResult(Result.UNSTABLE); + listener.getLogger().println(Messages.JUnitResultArchiver_ChangeState("UNSTABLE")); } public static TestResultAction parseAndAttach(@Nonnull JUnitTask task, PipelineTestDetails pipelineTestDetails, @@ -193,6 +213,12 @@ public static TestResultAction parseAndAttach(@Nonnull JUnitTask task, PipelineT // most likely a configuration error in the job - e.g. false pattern to match the JUnit result files throw new AbortException(Messages.JUnitResultArchiver_ResultIsEmpty()); } + + if (!task.isMakeUnstable()) { + // Change the buils state to Unstable if there are any failed test cases + listener.getLogger().println(Messages.JUnitResultArchiver_ResultIsEmpty()); + return null; + } // TODO: Move into JUnitParser [BUG 3123310] if (task.getTestDataPublishers() != null) { @@ -286,11 +312,23 @@ public boolean isKeepLongStdio() { public boolean isAllowEmptyResults() { return allowEmptyResults; } - + @DataBoundSetter public final void setAllowEmptyResults(boolean allowEmptyResults) { this.allowEmptyResults = allowEmptyResults; } + + /** + * + * @return the makeUnstable + */ + public boolean isMakeUnstable() { + return makeUnstable; + } + + @DataBoundSetter public final void setMakeUnstable(boolean makeUnstable) { + this.makeUnstable = makeUnstable; + } private static final long serialVersionUID = 1L; diff --git a/src/main/java/hudson/tasks/junit/JUnitTask.java b/src/main/java/hudson/tasks/junit/JUnitTask.java index 05590444d..8972d49e4 100644 --- a/src/main/java/hudson/tasks/junit/JUnitTask.java +++ b/src/main/java/hudson/tasks/junit/JUnitTask.java @@ -12,4 +12,6 @@ public interface JUnitTask { boolean isKeepLongStdio(); boolean isAllowEmptyResults(); + + boolean isMakeUnstable(); } diff --git a/src/main/java/hudson/tasks/junit/pipeline/JUnitResultsStep.java b/src/main/java/hudson/tasks/junit/pipeline/JUnitResultsStep.java index a730d840b..0aaf7d325 100644 --- a/src/main/java/hudson/tasks/junit/pipeline/JUnitResultsStep.java +++ b/src/main/java/hudson/tasks/junit/pipeline/JUnitResultsStep.java @@ -52,6 +52,7 @@ public class JUnitResultsStep extends Step implements JUnitTask { * If true, don't throw exception on missing test results or no files found. */ private boolean allowEmptyResults; + private boolean makeUnstable; @DataBoundConstructor public JUnitResultsStep(String testResults) { @@ -119,6 +120,19 @@ public boolean isAllowEmptyResults() { this.allowEmptyResults = allowEmptyResults; } + + /** + * + * @return the makeUnstable + */ + public boolean isMakeUnstable() { + return makeUnstable; + } + + @DataBoundSetter public final void setMakeUnstable(boolean makeUnstable) { + this.makeUnstable = makeUnstable; + } + @Override public StepExecution start(StepContext context) throws Exception { return new JUnitResultsStepExecution(this, context); diff --git a/src/main/resources/hudson/tasks/junit/JUnitResultArchiver/config.jelly b/src/main/resources/hudson/tasks/junit/JUnitResultArchiver/config.jelly index b7cc707f9..4649e92de 100644 --- a/src/main/resources/hudson/tasks/junit/JUnitResultArchiver/config.jelly +++ b/src/main/resources/hudson/tasks/junit/JUnitResultArchiver/config.jelly @@ -46,4 +46,7 @@ THE SOFTWARE. + + + diff --git a/src/main/resources/hudson/tasks/junit/JUnitResultArchiver/help-makeUnstable.html b/src/main/resources/hudson/tasks/junit/JUnitResultArchiver/help-makeUnstable.html new file mode 100644 index 000000000..77393d8da --- /dev/null +++ b/src/main/resources/hudson/tasks/junit/JUnitResultArchiver/help-makeUnstable.html @@ -0,0 +1,7 @@ +
+ If set to false, the default behavior of making the build unstable + for failed test result files is changed to not affect the status of the build. + Please note that this setting make it impossible to spot misconfigured jobs or + build failures where the test tool does not exit with an error code when + producing or not test report files. +
\ No newline at end of file diff --git a/src/main/resources/hudson/tasks/junit/Messages.properties b/src/main/resources/hudson/tasks/junit/Messages.properties index ee534eae2..8019c23bc 100644 --- a/src/main/resources/hudson/tasks/junit/Messages.properties +++ b/src/main/resources/hudson/tasks/junit/Messages.properties @@ -33,6 +33,7 @@ JUnitParser.TestResultLocationMessage=JUnit xml files: JUnitResultArchiver.DisplayName=Publish JUnit test result report JUnitResultArchiver.NoTestReportFound=No test report files were found. Configuration error? JUnitResultArchiver.Recording=Recording test results +JUnitResultArchiver.ChangeState=Changing build state to {0} because there are failed tests JUnitResultArchiver.ResultIsEmpty=None of the test reports contained any result JUnitResultArchiver.HealthScaleFactorAnalysis={0}% failing tests scores as {1}% health. {2}% failing tests scores as {3}% health diff --git a/src/test/java/hudson/tasks/junit/JUnitResultArchiverTest.java b/src/test/java/hudson/tasks/junit/JUnitResultArchiverTest.java index 1152a404b..fa104043b 100644 --- a/src/test/java/hudson/tasks/junit/JUnitResultArchiverTest.java +++ b/src/test/java/hudson/tasks/junit/JUnitResultArchiverTest.java @@ -313,6 +313,7 @@ public String getName() { @Test public void emptyDirectoryAllowEmptyResult() throws Exception { JUnitResultArchiver a = new JUnitResultArchiver("TEST-*.xml"); a.setAllowEmptyResults(true); + a.setMakeUnstable(true); FreeStyleProject freeStyleProject = j.createFreeStyleProject(); freeStyleProject.getPublishersList().add(a); j.assertBuildStatus(Result.SUCCESS, freeStyleProject.scheduleBuild2(0).get()); @@ -321,6 +322,7 @@ public String getName() { @Test public void emptyDirectory() throws Exception { JUnitResultArchiver a = new JUnitResultArchiver("TEST-*.xml"); a.setAllowEmptyResults(false); + a.setMakeUnstable(true); FreeStyleProject freeStyleProject = j.createFreeStyleProject(); freeStyleProject.getPublishersList().add(a); j.assertBuildStatus(Result.FAILURE, freeStyleProject.scheduleBuild2(0).get()); @@ -363,6 +365,7 @@ public void testDescribableRoundTrip() throws Exception { JUnitResultArchiver j = model.instantiate(args); assertEquals("**/TEST-*.xml", j.getTestResults()); assertFalse(j.isAllowEmptyResults()); + assertTrue(j.isMakeUnstable()); assertFalse(j.isKeepLongStdio()); assertEquals(1.0, j.getHealthScaleFactor(), 0); assertTrue(j.getTestDataPublishers().isEmpty()); diff --git a/src/test/java/hudson/tasks/junit/pipeline/JUnitResultsStepTest.java b/src/test/java/hudson/tasks/junit/pipeline/JUnitResultsStepTest.java index 8d94e093a..a05dc85f2 100644 --- a/src/test/java/hudson/tasks/junit/pipeline/JUnitResultsStepTest.java +++ b/src/test/java/hudson/tasks/junit/pipeline/JUnitResultsStepTest.java @@ -67,14 +67,15 @@ public class JUnitResultsStepTest { public void configRoundTrip() throws Exception { SnippetizerTester st = new SnippetizerTester(rule); JUnitResultsStep step = new JUnitResultsStep("**/target/surefire-reports/TEST-*.xml"); - st.assertRoundTrip(step, "junit '**/target/surefire-reports/TEST-*.xml'"); + step.setMakeUnstable(true); + st.assertRoundTrip(step, "junit makeUnstable: true, testResults: '**/target/surefire-reports/TEST-*.xml'"); step.setAllowEmptyResults(true); - st.assertRoundTrip(step, "junit allowEmptyResults: true, testResults: '**/target/surefire-reports/TEST-*.xml'"); + st.assertRoundTrip(step, "junit allowEmptyResults: true, makeUnstable: true, testResults: '**/target/surefire-reports/TEST-*.xml'"); step.setHealthScaleFactor(2.0); - st.assertRoundTrip(step, "junit allowEmptyResults: true, healthScaleFactor: 2.0, testResults: '**/target/surefire-reports/TEST-*.xml'"); + st.assertRoundTrip(step, "junit allowEmptyResults: true, healthScaleFactor: 2.0, makeUnstable: true, testResults: '**/target/surefire-reports/TEST-*.xml'"); MockTestDataPublisher publisher = new MockTestDataPublisher("testing"); step.setTestDataPublishers(Collections.singletonList(publisher)); - st.assertRoundTrip(step, "junit allowEmptyResults: true, healthScaleFactor: 2.0, testDataPublishers: [[$class: 'MockTestDataPublisher', name: 'testing']], testResults: '**/target/surefire-reports/TEST-*.xml'"); + st.assertRoundTrip(step, "junit allowEmptyResults: true, healthScaleFactor: 2.0, makeUnstable: true, testDataPublishers: [[$class: 'MockTestDataPublisher', name: 'testing']], testResults: '**/target/surefire-reports/TEST-*.xml'"); } @Issue("JENKINS-48250") @@ -119,7 +120,7 @@ public void allowEmpty() throws Exception { (Functions.isWindows() ? " bat 'echo hi'\n" : " sh 'echo hi'\n") + - " def results = junit(testResults: '*.xml', allowEmptyResults: true)\n" + + " def results = junit(testResults: '*.xml', makeUnstable: true, allowEmptyResults: true)\n" + " assert results.totalCount == 0\n" + " }\n" + "}\n", true)); @@ -134,7 +135,7 @@ public void singleStep() throws Exception { WorkflowJob j = rule.jenkins.createProject(WorkflowJob.class, "singleStep"); j.setDefinition(new CpsFlowDefinition("stage('first') {\n" + " node {\n" + - " def results = junit(testResults: '*.xml')\n" + // node id 7 + " def results = junit(testResults: '*.xml', makeUnstable: true)\n" + // node id 7 " assert results.totalCount == 6\n" + " }\n" + "}\n", true)); @@ -161,8 +162,8 @@ public void twoSteps() throws Exception { WorkflowJob j = rule.jenkins.createProject(WorkflowJob.class, "twoSteps"); j.setDefinition(new CpsFlowDefinition("stage('first') {\n" + " node {\n" + - " def first = junit(testResults: 'first-result.xml')\n" + // node id 7 - " def second = junit(testResults: 'second-result.xml')\n" + // node id 8 + " def first = junit(testResults: 'first-result.xml', makeUnstable: true)\n" + // node id 7 + " def second = junit(testResults: 'second-result.xml', makeUnstable: true)\n" + // node id 8 " assert first.totalCount == 6\n" + " assert second.totalCount == 1\n" + " }\n" + @@ -199,9 +200,9 @@ public void threeSteps() throws Exception { WorkflowJob j = rule.jenkins.createProject(WorkflowJob.class, "threeSteps"); j.setDefinition(new CpsFlowDefinition("stage('first') {\n" + " node {\n" + - " def first = junit(testResults: 'first-result.xml')\n" + // node id 7 - " def second = junit(testResults: 'second-result.xml')\n" + // node id 8 - " def third = junit(testResults: 'third-result.xml')\n" + // node id 9 + " def first = junit(testResults: 'first-result.xml', makeUnstable: true)\n" + // node id 7 + " def second = junit(testResults: 'second-result.xml', makeUnstable: true)\n" + // node id 8 + " def third = junit(testResults: 'third-result.xml', makeUnstable: true)\n" + // node id 9 " assert first.totalCount == 6\n" + " assert second.totalCount == 1\n" + " }\n" + @@ -255,9 +256,9 @@ public void parallelInStage() throws Exception { j.setDefinition(new CpsFlowDefinition("stage('first') {\n" + " node {\n" + - " parallel(a: { def first = junit(testResults: 'first-result.xml'); assert first.totalCount == 6 },\n" + - " b: { def second = junit(testResults: 'second-result.xml'); assert second.totalCount == 1 },\n" + - " c: { def third = junit(testResults: 'third-result.xml'); assert third.totalCount == 3 })\n" + + " parallel(a: { def first = junit(testResults: 'first-result.xml', makeUnstable: true); assert first.totalCount == 6 },\n" + + " b: { def second = junit(testResults: 'second-result.xml', makeUnstable: true); assert second.totalCount == 1 },\n" + + " c: { def third = junit(testResults: 'third-result.xml', makeUnstable: true); assert third.totalCount == 3 })\n" + " }\n" + "}\n", true )); @@ -288,9 +289,9 @@ public void stageInParallel() throws Exception { j.setDefinition(new CpsFlowDefinition("stage('outer') {\n" + " node {\n" + - " parallel(a: { stage('a') { def first = junit(testResults: 'first-result.xml'); assert first.totalCount == 6 } },\n" + - " b: { stage('b') { def second = junit(testResults: 'second-result.xml'); assert second.totalCount == 1 } },\n" + - " c: { stage('d') { def third = junit(testResults: 'third-result.xml'); assert third.totalCount == 3 } })\n" + + " parallel(a: { stage('a') { def first = junit(testResults: 'first-result.xml', makeUnstable: true); assert first.totalCount == 6 } },\n" + + " b: { stage('b') { def second = junit(testResults: 'second-result.xml', makeUnstable: true); assert second.totalCount == 1 } },\n" + + " c: { stage('d') { def third = junit(testResults: 'third-result.xml', makeUnstable: true); assert third.totalCount == 3 } })\n" + " }\n" + "}\n", true )); @@ -316,10 +317,10 @@ public void testTrends() throws Exception { WorkflowJob j = rule.jenkins.createProject(WorkflowJob.class, "testTrends"); j.setDefinition(new CpsFlowDefinition("node {\n" + " stage('first') {\n" + - " def first = junit(testResults: \"junit-report-testTrends-first.xml\")\n" + + " def first = junit(testResults: \"junit-report-testTrends-first.xml\", makeUnstable: true)\n" + " }\n" + " stage('second') {\n" + - " def second = junit(testResults: \"junit-report-testTrends-second.xml\")\n" + + " def second = junit(testResults: \"junit-report-testTrends-second.xml\", makeUnstable: true)\n" + " }\n" + "}\n", true)); FilePath ws = rule.jenkins.getWorkspaceFor(j); @@ -379,7 +380,7 @@ public void currentBuildResultUnstable() throws Exception { WorkflowJob j = rule.jenkins.createProject(WorkflowJob.class, "currentBuildResultUnstable"); j.setDefinition(new CpsFlowDefinition("stage('first') {\n" + " node {\n" + - " def results = junit(testResults: '*.xml')\n" + // node id 7 + " def results = junit(testResults: '*.xml', makeUnstable: true)\n" + // node id 7 " assert results.totalCount == 8\n" + " assert currentBuild.result == 'UNSTABLE'\n" + " }\n" +