diff --git a/README.md b/README.md index 2ac4488..1e304e9 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,7 @@ $ benchy --config test_data/config_test_load.yaml ``` ## Example Config -The config file can be either in JSON format or YAML. `benchy` assumes a file with the `yml` or `yaml` extension to be YAML, otherwise JSON is assumed. +The config file can be either in JSON format or YAML. `benchy` assumes a file with the `yml` or `yaml` extension to be YAML, otherwise JSON is assumed. More about configuration [here](docs/configuration.md). **Example YAML config:** ```yaml @@ -62,21 +62,20 @@ scenarios: workingDir: "/tmp" env: KEY: value - setup: + beforeAll: cmd: - echo - setupA - teardown: + afterAll: cmd: - echo - teardownA - - beforeCommand: + beforeEach: workingDir: "/another-path" cmd: - echo - beforeA - afterCommand: + afterEach: cmd: - echo - afterA @@ -103,26 +102,26 @@ scenarios: "env": { "KEY": "value" }, - "setup": { + "beforeAll": { "cmd": [ "echo", "setupA" ] }, - "teardown": { + "afterAll": { "cmd": [ "echo", "teardownA" ] }, - "beforeCommand": { + "beforeEach": { "workingDir": "/another-path", "cmd": [ "echo", "beforeA" ] }, - "afterCommand": { + "afterEach": { "cmd": [ "echo", "afterA" diff --git a/docs/configuration.md b/docs/configuration.md new file mode 100644 index 0000000..3325f52 --- /dev/null +++ b/docs/configuration.md @@ -0,0 +1,81 @@ +# Benchmark Configuration + +- [Benchmark Configuration](#benchmark-configuration) + - [Minimal Example](#minimal-example) + - [Full Example](#full-example) + - [Command Configuration](#command-configuration) + - [Alternate Execution](#alternate-execution) + +## Minimal Example +```yaml +--- +executions: 100 # required. number of times to execute each scenario +scenarios: # list of scenarios +- name: a scenario # required. unique scenario name + workingDir: "/tmp" # default working directory for commands executed in the context of this scenario + command: # required. the benchmarked command of this scenario - the one stats are collected for + cmd: # required. command line arguments. + - command + - --flag + - arg1 + - arg2 +``` + +## Full Example + +```yaml +--- +alternate: true # 'true' to alternate scenario executions. More details below. (default=false) +executions: 100 # required. number of times to execute each scenario +scenarios: # list of scenarios +- name: scenario A # required. unique scenario name + workingDir: "/tmp" # default working directory for commands executed in the context of this scenario + env: # environment variables to be set for commands executed in the context of this scenario + KEY: value + beforeAll: # command to be executed once before any other command is executed in the context of this scenario + cmd: # required. command line arguments. + - echo + - setupA + afterAll: # command to be executed once after all other commands in the context of this scenario + cmd: # required. command line arguments. + - echo + - teardownA + beforeEach: # command to be executed before each execution of this scenario + workingDir: "/path" # working directory only for this command + cmd: # required. command line arguments. + - echo + - beforeA + afterEach: # command to be executed after each execution of this scenario + cmd: # required. command line arguments. + - echo + - afterA + command: # required. the benchmarked command of this scenario - the one stats are collected for + cmd: # required. command line arguments. + - sleep + - '1' +- name: scenario B + command: + cmd: + - sleep + - '0' +``` + +## Command Configuration +The following elements share the same structure: `setupCommand`, `teardownCommand`, `beforeCommand`, `afterCommand`, `command`. + +`workingDir` - the `workingDir` property can be set globally for a scenario and optionally be overriden per command. If no working directory is set the default is the directory `benchy` is executed from. + +**Command structure:** +```yaml + command: # required. the benchmarked command of this scenario - the one stats are collected for + workingDir: "/path" # optional working directory for this command. + cmd: # required command line arguments. + - ls + - -l +``` + +## Alternate Execution +By default `benchy` executes scenarios in sequence and according to the number of `executions` set for your benchmark. Set the `alternate` property to `true` if you want spread the different scenarios more evenly over the time line. +Alternate execution can be helpful when: +- your benchmark runs for a very long time and external resources tend to behave differently over time +- you want some quiet time between executions of the same scenario to allow an extenral resource to cool down diff --git a/go.mod b/go.mod index 1acfc8d..c6c6dd9 100644 --- a/go.mod +++ b/go.mod @@ -4,9 +4,13 @@ go 1.15 require ( github.com/fatih/color v1.10.0 + github.com/go-playground/universal-translator v0.17.0 // indirect + github.com/leodido/go-urn v1.2.1 // indirect github.com/montanaflynn/stats v0.6.6 github.com/sirupsen/logrus v1.2.0 github.com/spf13/cobra v1.1.3 github.com/stretchr/testify v1.7.0 + gopkg.in/go-playground/assert.v1 v1.2.1 // indirect + gopkg.in/go-playground/validator.v9 v9.31.0 gopkg.in/yaml.v2 v2.4.0 ) diff --git a/go.sum b/go.sum index a9dc4ac..0d0a9a3 100644 --- a/go.sum +++ b/go.sum @@ -45,6 +45,10 @@ github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q= +github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= +github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no= +github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= @@ -108,6 +112,8 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w= +github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8= @@ -172,6 +178,7 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= @@ -294,6 +301,10 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/go-playground/assert.v1 v1.2.1 h1:xoYuJVE7KT85PYWrN730RguIQO0ePzVRfFMXadIrXTM= +gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= +gopkg.in/go-playground/validator.v9 v9.31.0 h1:bmXmP2RSNtFES+bn4uYuHT7iJFJv7Vj+an+ZQdDaD1M= +gopkg.in/go-playground/validator.v9 v9.31.0/go.mod h1:+c9/zcJMFNgbLvly1L1V+PpxWdVbfP1avr/N00E2vyQ= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= diff --git a/internal/benchmark_test.go b/internal/benchmark_test.go index b8b287a..9cb42ba 100644 --- a/internal/benchmark_test.go +++ b/internal/benchmark_test.go @@ -9,7 +9,7 @@ import ( var silentCommandExecutor = NewCommandExecutor(false, false) -func Test_run(t *testing.T) { +func TestRun(t *testing.T) { var actualSummary pkg.TracerSummary var actualConfig *BenchmarkSpec interceptSummary := func(summary pkg.TracerSummary, config *BenchmarkSpec) { @@ -29,7 +29,7 @@ func Test_run(t *testing.T) { assertFullScenarioStats(t, actualSummary.StatsOf("scenario B")) } -func Test_runWithMissingConfigFile(t *testing.T) { +func TestRunWithMissingConfigFile(t *testing.T) { err := run("../test_data/non-existing-file.yaml", silentCommandExecutor, failingWriteSummaryFn(t)) assert.Error(t, err) diff --git a/internal/exec.go b/internal/exec.go index ad39023..1e33a35 100644 --- a/internal/exec.go +++ b/internal/exec.go @@ -58,28 +58,28 @@ func executeSequencially(b *BenchmarkSpec, ctx *ExecutionContext) { func executeScenarioSetup(scenario *ScenarioSpec, ctx *ExecutionContext) { log.Debugf("Running setup for scenario '%s'...", scenario.Name) - ctx.executor.Execute(scenario.Setup, scenario.WorkingDirectory, scenario.Env, ctx) + ctx.executor.Execute(scenario.BeforeAll, scenario.WorkingDirectory, scenario.Env, ctx) } func executeScenarioTeardown(scenario *ScenarioSpec, ctx *ExecutionContext) { log.Debugf("Running teardown for scenario '%s'...", scenario.Name) - ctx.executor.Execute(scenario.Teardown, scenario.WorkingDirectory, scenario.Env, ctx) + ctx.executor.Execute(scenario.AfterAll, scenario.WorkingDirectory, scenario.Env, ctx) } func executeScenarioCommand(scenario *ScenarioSpec, ctx *ExecutionContext) { log.Infof("Executing scenario '%s'...", scenario.Name) - if scenario.BeforeCommand != nil { - log.Debugf("Executing 'before' command %v", scenario.BeforeCommand.Cmd) - ctx.executor.Execute(scenario.BeforeCommand, scenario.WorkingDirectory, scenario.Env, ctx) + if scenario.BeforeEach != nil { + log.Debugf("Executing 'before' command %v", scenario.BeforeEach.Cmd) + ctx.executor.Execute(scenario.BeforeEach, scenario.WorkingDirectory, scenario.Env, ctx) } ctx.tracer.Start(scenario)( ctx.executor.Execute(scenario.Command, scenario.WorkingDirectory, scenario.Env, ctx), ) - if scenario.AfterCommand != nil { - log.Debugf("Executing 'after' command %v", scenario.AfterCommand.Cmd) - ctx.executor.Execute(scenario.AfterCommand, scenario.WorkingDirectory, scenario.Env, ctx) + if scenario.AfterEach != nil { + log.Debugf("Executing 'after' command %v", scenario.AfterEach.Cmd) + ctx.executor.Execute(scenario.AfterEach, scenario.WorkingDirectory, scenario.Env, ctx) } } diff --git a/internal/spec.go b/internal/spec.go index 07c0069..9e55eea 100644 --- a/internal/spec.go +++ b/internal/spec.go @@ -2,6 +2,7 @@ package internal import ( "encoding/json" + "gopkg.in/go-playground/validator.v9" "gopkg.in/yaml.v2" "io/ioutil" "os" @@ -10,23 +11,23 @@ import ( type CommandSpec struct { WorkingDirectory string `json:"workingDir" yaml:"workingDir"` - Cmd []string `json:"cmd" yaml:"cmd" binding:"required"` + Cmd []string `json:"cmd" yaml:"cmd" validate:"required"` } type ScenarioSpec struct { - Name string `json:"name" yaml:"name" binding:"required"` + Name string `json:"name" yaml:"name" validate:"required"` WorkingDirectory string `json:"workingDir" yaml:"workingDir"` Env map[string]string - Setup *CommandSpec - Teardown *CommandSpec - BeforeCommand *CommandSpec `json:"beforeCommand" yaml:"beforeCommand" binding:"required"` - AfterCommand *CommandSpec `json:"aftercommand" yaml:"afterCommand" binding:"required"` - Command *CommandSpec `json:"command" yaml:"command" binding:"required"` + BeforeAll *CommandSpec `json:"beforeAll" yaml:"beforeAll"` + AfterAll *CommandSpec `json:"afterAll" yaml:"afterAll"` + BeforeEach *CommandSpec `json:"beforeEach" yaml:"beforeEach"` + AfterEach *CommandSpec `json:"afterEach" yaml:"afterEach"` + Command *CommandSpec `json:"command" yaml:"command" validate:"required"` } type BenchmarkSpec struct { - Scenarios []*ScenarioSpec `json:"scenarios" yaml:"scenarios" binding:"required"` - Executions int + Scenarios []*ScenarioSpec `json:"scenarios" yaml:"scenarios" validate:"required"` + Executions int `validate:"gte=1,required"` Alternate bool } @@ -34,15 +35,22 @@ func (s *ScenarioSpec) Id() string { return s.Name } -func Load(path string) (*BenchmarkSpec, error) { +func Load(path string) (rtn *BenchmarkSpec, err error) { if strings.HasSuffix(path, ".yaml") || strings.HasSuffix(path, ".yml") { - return load_yaml(path) + rtn, err = loadYaml(path) } else { - return load_json(path) + rtn, err = loadJson(path) } + + if err == nil { + v := validator.New() + err = v.Struct(rtn) + } + + return rtn, err } -func load_json(path string) (*BenchmarkSpec, error) { +func loadJson(path string) (*BenchmarkSpec, error) { var benchmark BenchmarkSpec jsonFile, err := os.Open(path) @@ -57,17 +65,15 @@ func load_json(path string) (*BenchmarkSpec, error) { return &benchmark, err } -func load_yaml(path string) (*BenchmarkSpec, error) { - var benchmark BenchmarkSpec - +func loadYaml(path string) (rtn *BenchmarkSpec, err error) { jsonFile, err := os.Open(path) if err == nil { defer jsonFile.Close() bytes, _ := ioutil.ReadAll(jsonFile) - yaml.Unmarshal(bytes, &benchmark) + yaml.Unmarshal(bytes, &rtn) } - return &benchmark, err + return rtn, err } diff --git a/internal/spec_test.go b/internal/spec_test.go index f9e42e2..44aff51 100644 --- a/internal/spec_test.go +++ b/internal/spec_test.go @@ -5,15 +5,15 @@ import ( "testing" ) -func Test_LoadJson(t *testing.T) { - test_Load(t, "../test_data/spec_test_load.json") +func TestLoadJson(t *testing.T) { + testLoad(t, "../test_data/spec_test_load.json") } -func Test_LoadYaml(t *testing.T) { - test_Load(t, "../test_data/spec_test_load.yaml") +func TestLoadYaml(t *testing.T) { + testLoad(t, "../test_data/spec_test_load.yaml") } -func test_Load(t *testing.T, filePath string) { +func testLoad(t *testing.T, filePath string) { expected := expectedBenchmarkSpec() benchmark, err := Load(filePath) @@ -31,17 +31,17 @@ func expectedBenchmarkSpec() *BenchmarkSpec { Name: "scenario A", WorkingDirectory: "/tmp", Env: map[string]string{"KEY": "value"}, - Setup: &CommandSpec{ + BeforeAll: &CommandSpec{ Cmd: []string{"echo", "setupA"}, }, - Teardown: &CommandSpec{ + AfterAll: &CommandSpec{ Cmd: []string{"echo", "teardownA"}, }, - BeforeCommand: &CommandSpec{ + BeforeEach: &CommandSpec{ WorkingDirectory: "/another-path", Cmd: []string{"echo", "beforeA"}, }, - AfterCommand: &CommandSpec{ + AfterEach: &CommandSpec{ Cmd: []string{"echo", "afterA"}, }, Command: &CommandSpec{ diff --git a/test_data/spec_test_load.json b/test_data/spec_test_load.json index bca816c..d7d3fe7 100644 --- a/test_data/spec_test_load.json +++ b/test_data/spec_test_load.json @@ -8,26 +8,26 @@ "env": { "KEY": "value" }, - "setup": { + "beforeAll": { "cmd": [ "echo", "setupA" ] }, - "teardown": { + "afterAll": { "cmd": [ "echo", "teardownA" ] }, - "beforeCommand": { + "beforeEach": { "workingDir": "/another-path", "cmd": [ "echo", "beforeA" ] }, - "afterCommand": { + "afterEach": { "cmd": [ "echo", "afterA" diff --git a/test_data/spec_test_load.yaml b/test_data/spec_test_load.yaml index 5016bc4..74d6a59 100644 --- a/test_data/spec_test_load.yaml +++ b/test_data/spec_test_load.yaml @@ -6,20 +6,20 @@ scenarios: workingDir: "/tmp" env: KEY: value - setup: + beforeAll: cmd: - echo - setupA - teardown: + afterAll: cmd: - echo - teardownA - beforeCommand: + beforeEach: workingDir: "/another-path" cmd: - echo - beforeA - afterCommand: + afterEach: cmd: - echo - afterA