fix: create empty lists in stats (#120)

* fix: create empty lists in stats

* fix: address comments. address golang-ci errors

Signed-off-by: Felipe Zipitria <[email protected]>

cmd/run.go

@@ -1,12 +1,12 @@
 package cmd
 
 import (
+	"errors"
 	"fmt"
 	"os"
 	"regexp"
 	"time"
 
-	"github.com/rs/zerolog/log"
 	"github.com/spf13/cobra"
 
 	"github.com/coreruleset/go-ftw/output"
@@ -19,7 +19,8 @@ var runCmd = &cobra.Command{
 	Use:   "run",
 	Short: "Run Tests",
 	Long:  `Run all tests below a certain subdirectory. The command will search all y[a]ml files recursively and pass it to the test engine.`,
-	Run: func(cmd *cobra.Command, args []string) {
+	RunE: func(cmd *cobra.Command, args []string) error {
+		cmd.SilenceUsage = true
 		exclude, _ := cmd.Flags().GetString("exclude")
 		include, _ := cmd.Flags().GetString("include")
 		id, _ := cmd.Flags().GetString("id")
@@ -36,16 +37,18 @@ var runCmd = &cobra.Command{
 			wantedOutput = "normal"
 		}
 		if id != "" {
-			log.Fatal().Msgf("--id is deprecated in favour of --include|-i")
+			cmd.SilenceUsage = false
+			return errors.New("--id is deprecated in favour of --include|-i")
 		}
 		if exclude != "" && include != "" {
-			log.Fatal().Msgf("You need to choose one: use --include (%s) or --exclude (%s)", include, exclude)
+			cmd.SilenceUsage = false
+			return fmt.Errorf("You need to choose one: use --include (%s) or --exclude (%s)", include, exclude)
 		}
 		files := fmt.Sprintf("%s/**/*.yaml", dir)
 		tests, err := test.GetTestsFromFiles(files)
 
 		if err != nil {
-			log.Fatal().Err(err)
+			return err
 		}
 
 		var includeRE *regexp.Regexp
@@ -71,11 +74,14 @@ var runCmd = &cobra.Command{
 			MaxMarkerRetries:  maxMarkerRetries,
 			MaxMarkerLogLines: maxMarkerLogLines,
 		}, out)
+
 		if err != nil {
-			log.Fatal().Err(err)
+			return err
 		}
-
-		os.Exit(currentRun.Stats.TotalFailed())
+		if currentRun.Stats.TotalFailed() > 0 {
+			return fmt.Errorf("failed %d tests", currentRun.Stats.TotalFailed())
+		}
+		return nil
 	},
 }
 

go.sum

@@ -77,8 +77,6 @@ github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/me
 github.com/go-test/deep v1.0.2-0.20181118220953-042da051cf31/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA=
 github.com/goccy/go-yaml v1.8.10 h1:XpBOLD8cmOZlLYjUFPqSZZ+Ubi4/UKxO2eXyhg5WuAA=
 github.com/goccy/go-yaml v1.8.10/go.mod h1:U/jl18uSupI5rdI2jmuCswEA2htH9eXfferR3KfscvA=
-github.com/goccy/go-yaml v1.9.6 h1:KhAu1zf9JXnm3vbG49aDE0E5uEBUsM4uwD31/58ZWyI=
-github.com/goccy/go-yaml v1.9.6/go.mod h1:JubOolP3gh0HpiBc4BLRD4YmjEjHAmIIB2aaXKkTfoE=
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
@@ -374,11 +372,8 @@ golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220406163625-3f8b81556e12/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U=
-golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.2.0 h1:ljd4t30dBnAvMZaQCevtY0xLLD0A+bRZXbgLMLU1F/A=
 golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=

output/output.go

@@ -90,6 +90,7 @@ func NewOutput(o string, w io.Writer) *Output {
 	log.Trace().Msgf("ftw/output: creating output %s\n", o)
 	out := &Output{
 		OutputType: Normal,
+		cat:        normalCatalog,
 		w:          w,
 	}
 	switch strings.ToLower(o) {
@@ -100,6 +101,7 @@ func NewOutput(o string, w io.Writer) *Output {
 	case "json":
 		out.OutputType = JSON
 	case "plain":
+		out.cat = createPlainCatalog(normalCatalog)
 		out.OutputType = Plain
 	case "normal":
 		break
@@ -112,25 +114,21 @@ func NewOutput(o string, w io.Writer) *Output {
 // Message predefined messages that might have different types depending on the output type.
 // All message in catalogs, where the text in the message is used as a key to get the corresponding text.
 func (o *Output) Message(key string) string {
-	var found bool
-	var text string
-	switch o.OutputType {
-	case Normal:
-		text, found = normalCatalog[key]
-		if !found {
-			text = ""
-		}
-	default:
-		text, found = normalCatalog[key]
-		if found {
-			text = key
-		} else {
-			text = ""
-		}
+	text, found := o.cat[key]
+	if !found {
+		text = ""
 	}
 	return text
 }
 
 func (o *Output) IsJson() bool {
 	return o.OutputType == JSON
 }
+
+func createPlainCatalog(c catalog) catalog {
+	plainCatalog := catalog{}
+	for k := range c {
+		plainCatalog[k] = k
+	}
+	return plainCatalog
+}

output/output_test.go

@@ -36,3 +36,31 @@ func TestOutput(t *testing.T) {
 		}
 	}
 }
+
+func TestNormalCatalogOutput(t *testing.T) {
+	var b bytes.Buffer
+
+	normal := NewOutput("normal", &b)
+	for _, v := range normalCatalog {
+		normal.RawPrint(v)
+		if b.String() != v {
+			t.Error("output is not equal")
+		}
+		// reset buffer
+		b.Reset()
+	}
+}
+
+func TestPlainCatalogOutput(t *testing.T) {
+	var b bytes.Buffer
+
+	normal := NewOutput("normal", &b)
+	for _, v := range createPlainCatalog(normalCatalog) {
+		normal.RawPrint(v)
+		if b.String() != v {
+			t.Error("plain output is not equal")
+		}
+		// reset buffer
+		b.Reset()
+	}
+}

runner/run.go

@@ -21,14 +21,12 @@ import (
 var errBadTestRequest = errors.New("ftw/run: bad test: choose between data, encoded_request, or raw_request")
 
 // Run runs your tests with the specified Config.
-func Run(tests []test.FTWTest, c Config, out *output.Output) (TestRunContext, error) {
+func Run(tests []test.FTWTest, c Config, out *output.Output) (*TestRunContext, error) {
 	out.Println("%s", out.Message("** Running go-ftw!"))
 
-	stats := NewRunStats()
-
 	logLines, err := waflog.NewFTWLogLines(waflog.WithLogFile(config.FTWConfig.LogFile))
 	if err != nil {
-		return TestRunContext{}, err
+		return &TestRunContext{}, err
 	}
 
 	conf := ftwhttp.NewClientConfig()
@@ -40,7 +38,7 @@ func Run(tests []test.FTWTest, c Config, out *output.Output) (TestRunContext, er
 	}
 	client, err := ftwhttp.NewClient(conf)
 	if err != nil {
-		return TestRunContext{}, err
+		return &TestRunContext{}, err
 	}
 	// TODO: These defaults shouldn't be initialized here but config intialization
 	// needs to be cleaned up properly first (e.g., with a `NewConfig()` function)
@@ -60,23 +58,23 @@ func Run(tests []test.FTWTest, c Config, out *output.Output) (TestRunContext, er
 		ShowOnlyFailed:    c.ShowOnlyFailed,
 		MaxMarkerRetries:  maxMarkerRetries,
 		MaxMarkerLogLines: maxMarkerLogLines,
-		Stats:             stats,
+		Stats:             NewRunStats(),
 		Client:            client,
 		LogLines:          logLines,
 		RunMode:           config.FTWConfig.RunMode,
 	}
 
 	for _, tc := range tests {
 		if err := RunTest(&runContext, tc); err != nil {
-			return TestRunContext{}, err
+			return &TestRunContext{}, err
 		}
 	}
 
 	runContext.Stats.printSummary(out)
 
 	defer cleanLogs(logLines)
 
-	return runContext, nil
+	return &runContext, nil
 }
 
 // RunTest runs an individual test.

runner/stats.go

@@ -44,12 +44,12 @@ type RunStats struct {
 func NewRunStats() *RunStats {
 	return &RunStats{
 		Run:        0,
-		Success:    nil,
-		Failed:     nil,
-		Skipped:    nil,
-		Ignored:    nil,
-		ForcedPass: nil,
-		ForcedFail: nil,
+		Success:    []string{},
+		Failed:     []string{},
+		Skipped:    []string{},
+		Ignored:    []string{},
+		ForcedPass: []string{},
+		ForcedFail: []string{},
 		RunTime:    make(map[string]time.Duration),
 		TotalTime:  0,
 	}