introducing config helper

sha1n · May 22, 2021 · 062917e · 062917e
1 parent 8951376
commit 062917e
Show file tree

Hide file tree

Showing 13 changed files with 632 additions and 118 deletions.
diff --git a/.vscode/launch.json b/.vscode/launch.json
@@ -0,0 +1,49 @@
+{
+  // Use IntelliSense to learn about possible attributes.
+  // Hover to view descriptions of existing attributes.
+  // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+  "version": "0.2.0",
+  "configurations": [
+    {
+      "name": "Benchy (basic)",
+      "type": "go",
+      "request": "launch",
+      "mode": "auto",
+      "program": "${workspaceFolder}/cmd/benchy.go",
+      "args": [
+        "-c",
+        "${workspaceFolder}/test/data/spec_test_load.yaml"
+      ]
+    },
+    {
+      "name": "Benchy (csv/raw)",
+      "type": "go",
+      "request": "launch",
+      "mode": "auto",
+      "program": "${workspaceFolder}/cmd/benchy.go",
+      "args": [
+        "-c",
+        "${workspaceFolder}/test/data/spec_test_load.yaml",
+        "--format",
+        "csv/raw",
+        "--label=vscode"
+      ]
+    },
+    {
+      "name": "Benchy (md)",
+      "type": "go",
+      "request": "launch",
+      "mode": "auto",
+      "program": "${workspaceFolder}/cmd/benchy.go",
+      "args": [
+        "-c",
+        "${workspaceFolder}/test/data/spec_test_load.yaml",
+        "--pipe-stdout=false",
+        "--silent",
+        "--format",
+        "md",
+        "--label=vscode"
+      ]
+    }
+  ]
+}
diff --git a/README.md b/README.md
@@ -5,15 +5,16 @@
 [![Release Drafter](https://github.com/sha1n/benchy/actions/workflows/release-drafter.yml/badge.svg)](https://github.com/sha1n/benchy/actions/workflows/release-drafter.yml)
 
 # benchy
-`benchy` is a simple CLI benchmarking tool that allows you to easily compare key performance metrics of different CLI commands. It was developed very quickly for a very specific use-case I had, but it is already very useful and can easily evolve into something even better.
+`benchy` is a CLI benchmarking tool that allows you to easily compare performance metrics of different CLI commands. I developed this tool to benchmark and compare development tools and configurations on different environment setups and machine over time. It is designed to support complex scenarios that require high level of control and consistency.
+
 
 - [benchy](#benchy)
   - [Main Features](#main-features)
   - [Installing](#installing)
     - [Download a prebuilt binary](#download-a-prebuilt-binary)
     - [Build your own binary](#build-your-own-binary)
   - [Usage](#usage)
-  - [Config File](#config-file)
+  - [Configurartion](#configurartion)
   - [Report Formats](#report-formats)
 
 ## Main Features
@@ -51,102 +52,48 @@ benchy --config test/data/spec_test_load.yaml
 benchy --help   # for full options list
 ```
 
-## Config File
-The config file can be either in JSON format or YAML. `benchy` assumes a file with the `yml` or `yaml` extension to be YAML, otherwise JSON is assumed. More about configuration [here](docs/configuration.md).
-
-**YAML Example:**
-```yaml
----
-alternate: true
-executions: 10
-scenarios:
-- name: scenario A
-  workingDir: "/tmp"
-  env:
-    KEY: value
-  beforeAll:
-    cmd:
-    - echo
-    - setupA
-  afterAll:
-    cmd:
-    - echo
-    - teardownA
-  beforeEach:
-    workingDir: "~/tmp"
-    cmd:
-    - echo
-    - beforeA
-  afterEach:
-    cmd:
-    - echo
-    - afterA
-  command:
-    cmd:
-    - sleep
-    - '1'
-- name: scenario B
-  command:
-    cmd:
-    - sleep
-    - '0'
-```
+## Configurartion
+`benchy` reads benchmark specifications from a config file. The config file can be either in YAML or JSON. `benchy` assumes a file with the `yml` or `yaml` extension to be YAML, otherwise JSON is assumed. You may create a configuratio file manually or use the `config` command to interactively generate your configuration.
+
+More about configuration [here](docs/configuration.md).
+
+**Benchy Config Utility Sample** 
+```bash
+benchy config -o benchmark.yaml
+
+--------------------------------
+ BENCHMARK CONFIGURATION HELPER
+--------------------------------
+
+This tool is going to help you go through a benchmark configuration definition.
+
+* annotates required input
+? annotates optional input
+
+more here: https://github.com/sha1n/benchy/blob/master/docs/configuration.md
+
+--------------------------------
+
+number of executions *: 30
+alternate executions ?: true
+scenario name *: sleepy scenario
+working directory ?:
+define custom env vars? (y/n):
+add setup command? (y/n): y
+working directory ?:
+command line *: echo 'preparing bedroom'
+add teardown command? (y/n): n
+add before each command? (y/n): y
+working directory ?:
+command line *: echo 'going to sleep'
+add after each command? (y/n):
+benchmarked command:
+working directory ?:
+command line *: sleep 1
+add another scenario? (y/n):
+
 
-**Equivalent JSON Example:**
-```json
-{
-  "alternate": true,
-  "executions": 10,
-  "scenarios": [
-    {
-      "name": "scenario A",
-      "workingDir": "/tmp",
-      "env": {
-        "KEY": "value"
-      },
-      "beforeAll": {
-        "cmd": [
-          "echo",
-          "setupA"
-        ]
-      },
-      "afterAll": {
-        "cmd": [
-          "echo",
-          "teardownA"
-        ]
-      },
-      "beforeEach": {
-        "workingDir": "~/tmp",
-        "cmd": [
-          "echo",
-          "beforeA"
-        ]
-      },
-      "afterEach": {
-        "cmd": [
-          "echo",
-          "afterA"
-        ]
-      },
-      "command": {
-        "cmd": [
-          "sleep",
-          "1"
-        ]
-      }
-    },
-    {
-      "name": "scenario B",
-      "command": {
-        "cmd": [
-          "sleep",
-          "0"
-        ]
-      }
-    }
-  ]
-}
+Writing your configuration...
 ```
 
 ## Report Formats

diff --git a/api/spec.go b/api/spec.go
@@ -2,27 +2,27 @@ package api
 
 // CommandSpec benchmark command execution specs
 type CommandSpec struct {
-	WorkingDirectory string   `json:"workingDir" yaml:"workingDir"`
+	WorkingDirectory string   `json:"workingDir,omitempty" yaml:"workingDir,omitempty"`
 	Cmd              []string `json:"cmd" yaml:"cmd" validate:"required"`
 }
 
 // ScenarioSpec benchmark scenario specs
 type ScenarioSpec struct {
 	Name             string `json:"name" yaml:"name" validate:"required"`
-	WorkingDirectory string `json:"workingDir" yaml:"workingDir"`
+	WorkingDirectory string `json:"workingDir,omitempty" yaml:"workingDir,omitempty"`
 	Env              map[string]string
-	BeforeAll        *CommandSpec `json:"beforeAll" yaml:"beforeAll"`
-	AfterAll         *CommandSpec `json:"afterAll" yaml:"afterAll"`
-	BeforeEach       *CommandSpec `json:"beforeEach" yaml:"beforeEach"`
-	AfterEach        *CommandSpec `json:"afterEach" yaml:"afterEach"`
+	BeforeAll        *CommandSpec `json:"beforeAll,omitempty" yaml:"beforeAll,omitempty"`
+	AfterAll         *CommandSpec `json:"afterAll,omitempty" yaml:"afterAll,omitempty"`
+	BeforeEach       *CommandSpec `json:"beforeEach,omitempty" yaml:"beforeEach,omitempty"`
+	AfterEach        *CommandSpec `json:"afterEach,omitempty" yaml:"afterEach,omitempty"`
 	Command          *CommandSpec `validate:"required,dive"`
 }
 
 // BenchmarkSpec benchmark specs top level structure
 type BenchmarkSpec struct {
 	Scenarios  []*ScenarioSpec `json:"scenarios" yaml:"scenarios" validate:"required,min=1,dive"`
 	Executions int             `validate:"required,gte=1"`
-	Alternate  bool
+	Alternate  bool            `json:"alternate,omitempty" yaml:"alternate,omitempty"`
 }
 
 // ID returns a unique identifier

diff --git a/cmd/benchy.go b/cmd/benchy.go
@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"os"
 
+	"github.com/sha1n/benchy/cmd/config"
 	"github.com/sha1n/benchy/internal/cli"
 	"github.com/spf13/cobra"
 )
@@ -53,6 +54,9 @@ csv/raw - CSV in which each row represents a raw trace event. useful if you want
 
 	rootCmd.SetVersionTemplate(`{{printf "%s" .Version}}`)
 
+	// Subcommands
+	rootCmd.AddCommand(config.CreateConfigCommand())
+
 	if err := rootCmd.Execute(); err != nil {
 		os.Exit(1)
 	}

diff --git a/cmd/config/config.go b/cmd/config/config.go
@@ -0,0 +1,22 @@
+package config
+
+import (
+	"github.com/sha1n/benchy/internal/cli"
+	"github.com/spf13/cobra"
+)
+
+// CreateConfigCommand creates the 'config' sub command
+func CreateConfigCommand() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "config",
+		Long:  `Interactively walks through a benchmark configuration creation process`,
+		Short: `interactively creates a benchmark config`,
+		Run:   cli.CreateConfig,
+	}
+
+	cmd.Flags().StringP(cli.ArgNameOutputFile, "o", "", `output file path. Optional. Writes to stdout by default.`)
+
+	_ = cmd.MarkFlagFilename(cli.ArgNameOutputFile, "yml", "yaml")
+
+	return cmd
+}