Merge pull request #226 from buildkite/tet-434-display-report-at-the-…

…end-of-test-run

Display report after run

go.mod

@@ -12,12 +12,14 @@ require (
 require (
 	drjosh.dev/zzglob v0.4.0
 	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51
+	github.com/olekukonko/tablewriter v0.0.5
 	github.com/pact-foundation/pact-go/v2 v2.0.8
 	golang.org/x/sys v0.27.0
 )
 
 require (
 	github.com/hashicorp/logutils v1.0.0 // indirect
+	github.com/mattn/go-runewidth v0.0.9 // indirect
 	github.com/stretchr/testify v1.9.0 // indirect
 	golang.org/x/net v0.28.0 // indirect
 	golang.org/x/text v0.17.0 // indirect

go.sum

@@ -10,6 +10,10 @@ github.com/hashicorp/logutils v1.0.0 h1:dLEQVugN8vlakKOUE3ihGLTZJRB4j+M2cdTm/ORI
 github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs=
 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
+github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0=
+github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
+github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec=
+github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY=
 github.com/pact-foundation/pact-go/v2 v2.0.8 h1:j9s/tk46O5hpEEbYd0/QF9kQlQt/mu3HJrVJVeix54w=
 github.com/pact-foundation/pact-go/v2 v2.0.8/go.mod h1:/IAP9loNwPHWdZUrECAltM8p630NETHitNarJa/DkXU=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=

internal/runner/cypress.go

@@ -28,28 +28,28 @@ func NewCypress(c RunnerConfig) Cypress {
 		c.TestFilePattern = "**/*.cy.{js,jsx,ts,tsx}"
 	}
 
-	return Cypress{c}
+	return Cypress{
+		RunnerConfig: c,
+	}
 }
 
-func (c Cypress) Run(testCases []plan.TestCase, retry bool) (RunResult, error) {
+func (c Cypress) Run(result *RunResult, testCases []plan.TestCase, retry bool) error {
 	testPaths := make([]string, len(testCases))
 	for i, tc := range testCases {
 		testPaths[i] = tc.Path
 	}
 	cmdName, cmdArgs, err := c.commandNameAndArgs(c.TestCommand, testPaths)
 	if err != nil {
-		return RunResult{Status: RunStatusError}, fmt.Errorf("failed to build command: %w", err)
+		result.err = err
+		return fmt.Errorf("failed to build command: %w", err)
 	}
 
 	cmd := exec.Command(cmdName, cmdArgs...)
 
 	err = runAndForwardSignal(cmd)
 
-	if err != nil {
-		return RunResult{Status: RunStatusError}, err
-	}
-
-	return RunResult{Status: RunStatusPassed}, nil
+	result.err = err
+	return err
 }
 
 func (c Cypress) GetFiles() ([]string, error) {

internal/runner/cypress_test.go

@@ -21,18 +21,15 @@ func TestCypressRun(t *testing.T) {
 	testCases := []plan.TestCase{
 		{Path: "./cypress/e2e/passing_spec.cy.js"},
 	}
-	got, err := cypress.Run(testCases, false)
-
-	want := RunResult{
-		Status: RunStatusPassed,
-	}
+	result := NewRunResult([]plan.TestCase{})
+	err := cypress.Run(result, testCases, false)
 
 	if err != nil {
 		t.Errorf("Cypress.Run(%q) error = %v", testCases, err)
 	}
 
-	if diff := cmp.Diff(got, want); diff != "" {
-		t.Errorf("Cypress.Run(%q) diff (-got +want):\n%s", testCases, diff)
+	if result.Status() != RunStatusPassed {
+		t.Errorf("Cypress.Run(%q) RunResult.Status = %v, want %v", testCases, result.Status(), RunStatusPassed)
 	}
 }
 
@@ -45,16 +42,13 @@ func TestCypressRun_TestFailed(t *testing.T) {
 
 	testCases := []plan.TestCase{
 		{Path: "./cypress/e2e/failing_spec.cy.js"},
-        {Path: "./cypress/e2e/passing_spec.cy.js"},
-	}
-	got, err := cypress.Run(testCases, false)
-
-	want := RunResult{
-		Status: RunStatusError,
+		{Path: "./cypress/e2e/passing_spec.cy.js"},
 	}
+	result := NewRunResult([]plan.TestCase{})
+	err := cypress.Run(result, testCases, false)
 
-	if diff := cmp.Diff(got, want); diff != "" {
-		t.Errorf("Cypress.Run(%q) diff (-got +want):\n%s", testCases, diff)
+	if result.Status() != RunStatusError {
+		t.Errorf("Cypress.Run(%q) RunResult.Status = %v, want %v", testCases, result.Status(), RunStatusError)
 	}
 
 	exitError := new(exec.ExitError)
@@ -64,21 +58,16 @@ func TestCypressRun_TestFailed(t *testing.T) {
 }
 
 func TestCypressRun_CommandFailed(t *testing.T) {
-	cypress := Cypress{
-		RunnerConfig{
-			TestCommand: "yarn cypress run --json",
-		},
-	}
+	cypress := NewCypress(RunnerConfig{
+		TestCommand: "yarn cypress run --json",
+	})
 
 	testCases := []plan.TestCase{}
-	got, err := cypress.Run(testCases, false)
+	result := NewRunResult([]plan.TestCase{})
+	err := cypress.Run(result, testCases, false)
 
-	want := RunResult{
-		Status: RunStatusError,
-	}
-
-	if diff := cmp.Diff(got, want); diff != "" {
-		t.Errorf("Cypress.Run(%q) diff (-got +want):\n%s", testCases, diff)
+	if result.Status() != RunStatusError {
+		t.Errorf("Cypress.Run(%q) RunResult.Status = %v, want %v", testCases, result.Status(), RunStatusError)
 	}
 
 	exitError := new(exec.ExitError)
@@ -91,18 +80,15 @@ func TestCypressRun_SignaledError(t *testing.T) {
 	cypress := NewCypress(RunnerConfig{
 		TestCommand: "./testdata/segv.sh",
 	})
+
 	testCases := []plan.TestCase{
 		{Path: "./doesnt-matter.cy.js"},
 	}
+	result := NewRunResult([]plan.TestCase{})
+	err := cypress.Run(result, testCases, false)
 
-	got, err := cypress.Run(testCases, false)
-
-	want := RunResult{
-		Status: RunStatusError,
-	}
-
-	if diff := cmp.Diff(got, want); diff != "" {
-		t.Errorf("Cypress.Run(%q) diff (-got +want):\n%s", testCases, diff)
+	if result.Status() != RunStatusError {
+		t.Errorf("Cypress.Run(%q) RunResult.Status = %v, want %v", testCases, result.Status(), RunStatusError)
 	}
 
 	signalError := new(ProcessSignaledError)
@@ -137,12 +123,10 @@ func TestCypressCommandNameAndArgs_WithInterpolationPlaceholder(t *testing.T) {
 	testCases := []string{"cypress/e2e/passing_spec.cy.js", "cypress/e2e/flaky_spec.cy.js"}
 	testCommand := "cypress run --spec {{testExamples}}"
 
-	cy := Cypress{
-		RunnerConfig{
-			TestCommand: testCommand,
-			ResultPath:  "cypress.json",
-		},
-	}
+	cy := NewCypress(RunnerConfig{
+		TestCommand: testCommand,
+		ResultPath:  "cypress.json",
+	})
 
 	gotName, gotArgs, err := cy.commandNameAndArgs(testCommand, testCases)
 	if err != nil {
@@ -164,11 +148,9 @@ func TestCypressCommandNameAndArgs_WithoutTestExamplesPlaceholder(t *testing.T)
 	testCases := []string{"cypress/e2e/passing_spec.cy.js", "cypress/e2e/flaky_spec.cy.js"}
 	testCommand := "cypress run"
 
-	cypress := Cypress{
-		RunnerConfig{
-			TestCommand: testCommand,
-		},
-	}
+	cypress := NewCypress(RunnerConfig{
+		TestCommand: testCommand,
+	})
 
 	gotName, gotArgs, err := cypress.commandNameAndArgs(testCommand, testCases)
 	if err != nil {
@@ -190,11 +172,9 @@ func TestCypressCommandNameAndArgs_InvalidTestCommand(t *testing.T) {
 	testCases := []string{"cypress/e2e/passing_spec.cy.js", "cypress/e2e/flaky_spec.cy.js"}
 	testCommand := "cypress run --options '{{testExamples}}"
 
-	cypress := Cypress{
-		RunnerConfig{
-			TestCommand: testCommand,
-		},
-	}
+	cypress := NewCypress(RunnerConfig{
+		TestCommand: testCommand,
+	})
 
 	gotName, gotArgs, err := cypress.commandNameAndArgs(testCommand, testCases)
 

internal/runner/detector.go

@@ -17,7 +17,7 @@ type RunnerConfig struct {
 }
 
 type TestRunner interface {
-	Run(testCases []plan.TestCase, retry bool) (RunResult, error)
+	Run(result *RunResult, testCases []plan.TestCase, retry bool) error
 	GetExamples(files []string) ([]plan.TestCase, error)
 	GetFiles() ([]string, error)
 	Name() string

internal/runner/jest.go

@@ -32,7 +32,9 @@ func NewJest(j RunnerConfig) Jest {
 		j.RetryTestCommand = "npx jest --testNamePattern '{{testNamePattern}}' --json --testLocationInResults --outputFile {{resultPath}}"
 	}
 
-	return Jest{j}
+	return Jest{
+		RunnerConfig: j,
+	}
 }
 
 func (j Jest) Name() string {
@@ -56,7 +58,7 @@ func (j Jest) GetFiles() ([]string, error) {
 	return files, nil
 }
 
-func (j Jest) Run(testCases []plan.TestCase, retry bool) (RunResult, error) {
+func (j Jest) Run(result *RunResult, testCases []plan.TestCase, retry bool) error {
 	var cmd *exec.Cmd
 	var err error
 
@@ -67,7 +69,8 @@ func (j Jest) Run(testCases []plan.TestCase, retry bool) (RunResult, error) {
 		}
 		commandName, commandArgs, err := j.commandNameAndArgs(j.TestCommand, testPaths)
 		if err != nil {
-			return RunResult{Status: RunStatusError}, fmt.Errorf("failed to build command: %w", err)
+			result.err = err
+			return fmt.Errorf("failed to build command: %w", err)
 		}
 
 		cmd = exec.Command(commandName, commandArgs...)
@@ -78,50 +81,49 @@ func (j Jest) Run(testCases []plan.TestCase, retry bool) (RunResult, error) {
 		}
 		commandName, commandArgs, err := j.retryCommandNameAndArgs(j.RetryTestCommand, testNames)
 		if err != nil {
-			return RunResult{Status: RunStatusError}, fmt.Errorf("failed to build command: %w", err)
+			result.err = err
+			return fmt.Errorf("failed to build command: %w", err)
 		}
 
 		cmd = exec.Command(commandName, commandArgs...)
 	}
 
 	err = runAndForwardSignal(cmd)
 
-	if err == nil { // note: returning success early
-		return RunResult{Status: RunStatusPassed}, nil
-	}
-
 	if ProcessSignaledError := new(ProcessSignaledError); errors.As(err, &ProcessSignaledError) {
-		return RunResult{Status: RunStatusError}, err
+		result.err = err
+		return err
 	}
 
-	if exitError := new(exec.ExitError); errors.As(err, &exitError) {
-		report, parseErr := j.ParseReport(j.ResultPath)
-		if parseErr != nil {
-			fmt.Println("Buildkite Test Engine Client: Failed to read Jest output, tests will not be retried.")
-			return RunResult{Status: RunStatusError}, err
-		}
+	report, parseErr := j.ParseReport(j.ResultPath)
+	if parseErr != nil {
+		fmt.Println("Buildkite Test Engine Client: Failed to read Jest output, tests will not be retried.")
+		result.err = err
+		return err
+	}
 
-		if report.NumFailedTests > 0 {
-			var failedTests []plan.TestCase
-			for _, testResult := range report.TestResults {
-				for _, example := range testResult.AssertionResults {
-					if example.Status == "failed" {
-						failedTests = append(failedTests, plan.TestCase{
-							// The scope and name has to match with the scope generated by Buildkite test collector.
-							// For more details, see:
-							// [Buildkite Test Collector - Jest implementation](https://github.com/buildkite/test-collector-javascript/blob/42b803a618a15a07edf0169038ef4b5eba88f98d/jest/reporter.js#L40)
-							Name:  example.Title,
-							Scope: strings.Join(example.AncestorTitles, " "),
-						})
-					}
-				}
+	for _, testResult := range report.TestResults {
+		for _, example := range testResult.AssertionResults {
+			var status TestStatus
+			switch example.Status {
+			case "failed":
+				status = TestStatusFailed
+			case "passed":
+				status = TestStatusPassed
 			}
 
-			return RunResult{Status: RunStatusFailed, FailedTests: failedTests}, nil
+			// The scope and name has to match with the scope generated by Buildkite test collector.
+			// For more details, see:
+			// [Buildkite Test Collector - Jest implementation](https://github.com/buildkite/test-collector-javascript/blob/42b803a618a15a07edf0169038ef4b5eba88f98d/jest/reporter.js#L40)
+			testCase := plan.TestCase{
+				Name:  example.Title,
+				Scope: strings.Join(example.AncestorTitles, " "),
+			}
+			result.RecordTestResult(testCase, status)
 		}
 	}
 
-	return RunResult{Status: RunStatusError}, err
+	return nil
 }
 
 type JestExample struct {