introducing experimental terminal based gui (#84)

* introducing progress listener api

* working matrix view

* added docker based manual tests

.dockerignore

@@ -0,0 +1,3 @@
+vendor
+bin
+build

Dockerfile

@@ -0,0 +1,7 @@
+FROM golang:1.16
+
+ADD . /benchy
+
+WORKDIR /benchy
+
+RUN make go-build-linux-amd64

Makefile

@@ -112,6 +112,24 @@ go-clean:
 	@GOPATH=$(GOPATH) GOBIN=$(GOBIN) go clean $(MODFLAGS) $(GOBASE)/cmd
 	@GOPATH=$(GOPATH) GOBIN=$(GOBIN) go clean -modcache
 
+run-sanity-tests: build-docker run-linux-dockerized-tests
+
+build-docker:
+	@echo "  >  Building docker image..."
+	docker build -t sha1n/$(PROJECTNAME):latest .
+	docker tag sha1n/$(PROJECTNAME):latest sha1n/$(PROJECTNAME):$(VERSION:v%=%)
+
+run-linux-dockerized-tests:
+	@echo "  >  Running with experimental UI..."
+	docker run -ti sha1n/benchy /benchy/bin/benchy-linux-amd64 -c /benchy/test/data/spec_test_load.yaml --experimental ui
+	@echo "  >  Running with experimental UI + debug..."
+	docker run -ti sha1n/benchy /benchy/bin/benchy-linux-amd64 -c /benchy/test/data/spec_test_load.yaml --experimental ui -d
+	@echo "  >  Running with experimental UI + silent..."
+	docker run -ti sha1n/benchy /benchy/bin/benchy-linux-amd64 -c /benchy/test/data/spec_test_load.yaml --experimental ui -s
+	@echo "  >  Running with experimental UI + piped stdout..."
+	docker run -ti sha1n/benchy /benchy/bin/benchy-linux-amd64 -c /benchy/test/data/spec_test_load.yaml --experimental ui --pipe-stdout
+	@echo "  >  Running with experimental UI + piped stderr..."
+	docker run -ti sha1n/benchy /benchy/bin/benchy-linux-amd64 -c /benchy/test/data/spec_test_load.yaml --experimental ui --pipe-stderr
 
 
 .PHONY: help

api/exec.go

@@ -9,12 +9,14 @@ type CommandExecutor interface {
 type ExecutionContext struct {
 	Executor CommandExecutor
 	Tracer   Tracer
+	Listener
 }
 
 // NewExecutionContext creates a new ExecutionContext.
-func NewExecutionContext(tracer Tracer, executor CommandExecutor) ExecutionContext {
+func NewExecutionContext(tracer Tracer, executor CommandExecutor, listener Listener) ExecutionContext {
 	return ExecutionContext{
 		Executor: executor,
 		Tracer:   tracer,
+		Listener: listener,
 	}
 }

internal/cli/iocontext.go → api/iocontext.go

@@ -1,4 +1,4 @@
-package cli
+package api
 
 import (
 	"io"

api/listener.go

@@ -0,0 +1,12 @@
+package api
+
+// Listener a listener for benchmark progress events
+type Listener interface {
+	OnBenchmarkStart()
+	OnBenchmarkEnd()
+	OnScenarioStart(id ID)
+	OnScenarioEnd(id ID)
+	OnMessagef(id ID, format string, args ...interface{})
+	OnMessage(id ID, message string)
+	OnError(id ID, err error)
+}

cmd/main.go

@@ -5,6 +5,7 @@ import (
 	"runtime/debug"
 
 	"github.com/fatih/color"
+	"github.com/sha1n/benchy/api"
 	"github.com/sha1n/benchy/internal/cli"
 	errorhandling "github.com/sha1n/clib/pkg/error_handling"
 	log "github.com/sirupsen/logrus"
@@ -29,10 +30,10 @@ func main() {
 	doRun(doExit)
 }
 
-func doRun(exit func(int)) {
-	defer handlePanics(exit)
+func doRun(exitFn func(int)) {
+	defer handlePanics(exitFn)
 
-	ctx := cli.NewIOContext()
+	ctx := api.NewIOContext()
 	rootCmd := cli.NewRootCommand(ProgramName, Version, Build, ctx)
 
 	// Subcommands
@@ -44,22 +45,22 @@ func doRun(exit func(int)) {
 	}
 }
 
-func handlePanics(exit func(int)) {
+func handlePanics(exitFn func(int)) {
 	if o := recover(); o != nil {
-		if err, ok := o.(error); ok {
-			log.Error(err.Error())
-		} else {
-			log.Error(o)
+		if err, ok := o.(cli.FatalUserError); ok {
+			log.Fatal(err)
+			exitFn(1)
 		}
+
 		issueURL := errorhandling.GenerateGitHubCreateNewIssueURL(
 			"sha1n",
 			"benchy",
 			fmt.Sprintf("Panic Issue (%s, build: %s)", Version, Build),
-			fmt.Sprintf("```Captured error: %s```", debug.Stack()),
+			fmt.Sprintf("**Error:** %s\n**Stacktrace:**\n```%s```", o, debug.Stack()),
 		) + "&labels=bug"
 
 		yellow := color.New(color.FgYellow)
-		yellow.Println("\nPlease kindly report this issue by following this URL:")
+		yellow.Println("\nOh no... Please kindly report this issue by following this URL:")
 		fmt.Printf(`
 
 %s
@@ -68,7 +69,7 @@ func handlePanics(exit func(int)) {
 			issueURL,
 		)
 
-		exit(1)
+		exitFn(1)
 	}
 }
 

cmd/main_test.go

@@ -1,47 +1,57 @@
 package main
 
 import (
-	"bytes"
 	"os"
-	"os/exec"
 	"testing"
 
+	"github.com/sha1n/termite"
 	"github.com/stretchr/testify/assert"
 )
 
-var runMainCommand = []string{
-	"go",
-	"run",
-	"-mod=readonly",
-	"main.go",
-}
-
-func TestExitCodeWhenRequiredConfigArgIsMissing(t *testing.T) {
-	expectedExitCode := 1
-	buf := new(bytes.Buffer)
-
-	cmd := exec.Command(runMainCommand[0], runMainCommand[1:]...)
-
-	cmd.Stdout = buf
-	cmd.Stderr = buf
-
-	assert.NoError(t, cmd.Start())
-	state, err := cmd.Process.Wait()
-
-	assert.Contains(t, buf.String(), "Error: required flag(s) \"config\" not set")
-	assert.NoError(t, err)
-	assert.Equal(t, expectedExitCode, state.ExitCode())
-
-}
-
-func TestExitCodeWithMissingConfig(t *testing.T) {
-	expectedExitCode := 1
+func TestExitCodeWithMissingRequiredArguments(t *testing.T) {
+	expectedPanicExitCode := 1
 	actualExitcode := 0
 
 	os.Args = []string{}
 	doRun(func(i int) {
 		actualExitcode = i
 	})
 
-	assert.Equal(t, expectedExitCode, actualExitcode)
+	assert.Equal(t, expectedPanicExitCode, actualExitcode)
+}
+
+func TestSanity(t *testing.T) {
+	testWith(
+		t,
+		[]string{
+			"program",
+			"-c",
+			"../test/data/integration.yaml",
+			"--debug",
+			"--pipe-stdout",
+			"--pipe-stderr",
+		},
+		true,
+		func(t *testing.T) {
+			doRun(func(code int) {
+				assert.Equal(t, 0, code)
+			})
+		},
+	)
+}
+
+func testWith(t *testing.T, args []string, tty bool, test func(t *testing.T)) {
+	origTtyValue := termite.Tty
+	origOsArgs := os.Args
+	termite.Tty = true
+
+	defer func() {
+		termite.Tty = origTtyValue
+		os.Args = origOsArgs
+	}()
+
+	os.Args = args
+	termite.Tty = tty
+
+	test(t)
 }

go.mod

@@ -10,7 +10,7 @@ require (
 	github.com/leodido/go-urn v1.2.1 // indirect
 	github.com/montanaflynn/stats v0.6.6
 	github.com/sha1n/clib v0.0.6
-	github.com/sha1n/termite v0.0.9
+	github.com/sha1n/termite v0.0.14
 	github.com/sirupsen/logrus v1.8.1
 	github.com/spf13/cobra v1.1.3
 	github.com/stretchr/testify v1.7.0

go.sum

@@ -169,12 +169,10 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
-github.com/sha1n/clib v0.0.5 h1:4x0iKGi4XVHkzaoA71QbgsuHUGIcIsi+V1cgvIyLKIo=
-github.com/sha1n/clib v0.0.5/go.mod h1:772YEqQrJTOMl+Cl1wxPbjyhsVDLFb8Q2dpkBJIrzgw=
 github.com/sha1n/clib v0.0.6 h1:ntG94hgD6+pRGXuP8nPbmvcd7UWdGjyQYV3QenXB4fw=
 github.com/sha1n/clib v0.0.6/go.mod h1:772YEqQrJTOMl+Cl1wxPbjyhsVDLFb8Q2dpkBJIrzgw=
-github.com/sha1n/termite v0.0.9 h1:KBR35vWGi00XBf21AOf7fZxNVWu8YBJmBEv1yXwextM=
-github.com/sha1n/termite v0.0.9/go.mod h1:ONnG2xokk/bKYzSapiv2WZ17Fv3OnsNTZOyEtxwEyb4=
+github.com/sha1n/termite v0.0.14 h1:g9jagZWWUJyAu1cHH9QFiZGEQ+xJ9QfzkxER+qcbkYY=
+github.com/sha1n/termite v0.0.14/go.mod h1:ONnG2xokk/bKYzSapiv2WZ17Fv3OnsNTZOyEtxwEyb4=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=

internal/cli/args.go

@@ -6,6 +6,7 @@ import (
 	"path/filepath"
 	"strings"
 
+	"github.com/sha1n/benchy/api"
 	log "github.com/sirupsen/logrus"
 
 	"github.com/spf13/cobra"
@@ -48,9 +49,12 @@ const (
 	ArgValueReportFormatMarkdownRaw = "md/raw"
 )
 
+// StreamingReportFormats a slice containing the values that represent report formats that are reporting in streaming
+var StreamingReportFormats = map[string]bool{ArgValueReportFormatCsvRaw: true, ArgValueReportFormatMarkdownRaw: true}
+
 // ResolveOutputArg resolves an output file argument based on user input.
 // If the specified argument is empty, stdout is returned.
-func ResolveOutputArg(cmd *cobra.Command, name string, ctx IOContext) io.WriteCloser {
+func ResolveOutputArg(cmd *cobra.Command, name string, ctx api.IOContext) io.WriteCloser {
 	var outputFile io.WriteCloser = stdOutNonClosingWriteCloser{out: ctx.StdoutWriter}
 	var err error = nil
 

internal/cli/args_test.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"testing"
 
+	"github.com/sha1n/benchy/api"
 	clibtest "github.com/sha1n/clib/pkg/test"
 	"github.com/spf13/cobra"
 	"github.com/stretchr/testify/assert"
@@ -45,7 +46,7 @@ func experimentalFlagWith(value string) string {
 }
 
 func withCommandWithArgs(t *testing.T, doTest func(cmd *cobra.Command), args ...string) {
-	ctx := NewIOContext()
+	ctx := api.NewIOContext()
 	cmd := NewRootCommand(clibtest.RandomString(), clibtest.RandomString(), clibtest.RandomString(), ctx)
 	cmd.SetArgs(append(args, "--config=xxx"))
 	cmd.Run = func(c *cobra.Command, args []string) {

internal/cli/config_runner.go

@@ -11,7 +11,7 @@ import (
 )
 
 // CreateConfigCommand creates the 'config' sub command
-func CreateConfigCommand(ctx IOContext) *cobra.Command {
+func CreateConfigCommand(ctx api.IOContext) *cobra.Command {
 	cmd := &cobra.Command{
 		Use:   "config",
 		Long:  `Interactively walks through a benchmark configuration creation process`,
@@ -27,7 +27,7 @@ func CreateConfigCommand(ctx IOContext) *cobra.Command {
 }
 
 // createConfigFn returns a function that runs the config tool with the specified IOContext
-func createConfigFn(ctx IOContext) func(*cobra.Command, []string) {
+func createConfigFn(ctx api.IOContext) func(*cobra.Command, []string) {
 	return func(cmd *cobra.Command, args []string) {
 		configureOutput(cmd, ctx)
 
@@ -50,7 +50,7 @@ func createConfigFn(ctx IOContext) func(*cobra.Command, []string) {
 	}
 }
 
-func requestScenarios(ctx IOContext) []api.ScenarioSpec {
+func requestScenarios(ctx api.IOContext) []api.ScenarioSpec {
 	specs := []api.ScenarioSpec{}
 
 	for {
@@ -63,7 +63,7 @@ func requestScenarios(ctx IOContext) []api.ScenarioSpec {
 	return specs
 }
 
-func requestCommand(description string, required bool, ctx IOContext) *api.CommandSpec {
+func requestCommand(description string, required bool, ctx api.IOContext) *api.CommandSpec {
 	requestCommand := func() *api.CommandSpec {
 		return &api.CommandSpec{
 			WorkingDirectory: requestOptionalExistingDirectory("working directory", "inherits scenario", ctx),
@@ -82,7 +82,7 @@ func requestCommand(description string, required bool, ctx IOContext) *api.Comma
 	return nil
 }
 
-func requestEnvVars(ctx IOContext) map[string]string {
+func requestEnvVars(ctx api.IOContext) map[string]string {
 	var envVars map[string]string
 
 	if questionYN("define custom env vars?", ctx) {
@@ -101,7 +101,7 @@ func requestEnvVars(ctx IOContext) map[string]string {
 	return envVars
 }
 
-func requestScenario(ctx IOContext) api.ScenarioSpec {
+func requestScenario(ctx api.IOContext) api.ScenarioSpec {
 	return api.ScenarioSpec{
 		Name:             requestString("scenario name", true, ctx),
 		WorkingDirectory: requestOptionalExistingDirectory("working directory", "inherits current", ctx),

internal/cli/config_runner_test.go

@@ -39,7 +39,7 @@ func TestBasicInteractiveFlow(t *testing.T) {
 	assert.Equal(t, expectedSpec(), actual)
 }
 
-func configureCommand(t *testing.T, ctx IOContext) (command *cobra.Command, configPath string, teardown func()) {
+func configureCommand(t *testing.T, ctx api.IOContext) (command *cobra.Command, configPath string, teardown func()) {
 	rootCmd := NewRootCommand(clibtest.RandomString(), clibtest.RandomString(), clibtest.RandomString(), ctx)
 	cmd := CreateConfigCommand(ctx)
 	rootCmd.AddCommand(cmd)