From d660414a9d8b3c3c68168841de62d33971f89937 Mon Sep 17 00:00:00 2001 From: Syakir Date: Wed, 3 Feb 2021 02:10:34 +0000 Subject: [PATCH 1/7] chore: Fix linting --- scenario.go | 50 ++++++++++++++++++++++++++++---------------------- 1 file changed, 28 insertions(+), 22 deletions(-) diff --git a/scenario.go b/scenario.go index 068a9bc..dfa1712 100644 --- a/scenario.go +++ b/scenario.go @@ -18,27 +18,32 @@ import ( "gopkg.in/yaml.v2" ) +//Asserts represents acceptance criteria for a test case type Asserts struct { Code int `yaml:"status_code"` ValidateJSON string `yaml:"validate_json"` Script string `yaml:"script"` } -type RunHttp struct { +//RunHTTP represents configuration on how to run HTTP test +type RunHTTP struct { Method string `yaml:"method"` - Url string `yaml:"url"` + URL string `yaml:"url"` Headers map[string]string `yaml:"headers"` QueryParams map[string]string `yaml:"query_params"` + Files map[string]string `yaml:"files"` Forms map[string]string `yaml:"forms"` Payload string `yaml:"payload"` ResponseOut string `yaml:"response_out"` Asserts *Asserts `yaml:"asserts"` } +//Run represent methods for testing type Run struct { - Http RunHttp `yaml:"http"` + HTTP RunHTTP `yaml:"http"` } +//ReportPubsub represents configuration to report to pubsub type ReportPubsub struct { Scenario string `json:"scenario"` Attributes map[string]string `json:"attributes"` // [status]=success|error @@ -46,7 +51,7 @@ type ReportPubsub struct { Data string `json:"data"` } -// Scenario reprents a single scenario file to run. +// Scenario represents a single scenario file to run. type Scenario struct { Tags map[string]string `yaml:"tags"` Env map[string]string `yaml:"env"` @@ -141,11 +146,12 @@ func (s *Scenario) WriteScript(file, contents string) (string, error) { return file, err } -// LoggerReporter interface for httpexpect. +//Logf interface for httpexpect. func (s Scenario) Logf(fmt string, args ...interface{}) { log.Printf(fmt, args...) } +//Errorf returns formatted error message func (s Scenario) Errorf(message string, args ...interface{}) { m := fmt.Sprintf(message, args...) s.me.errs = append(s.me.errs, fmt.Errorf(m)) @@ -226,9 +232,9 @@ func doScenario(in *doScenarioInput) error { // Parse url. fn := fmt.Sprintf("%v_url", prefix) - nv, err := s.ParseValue(run.Http.Url, fn) + nv, err := s.ParseValue(run.HTTP.URL, fn) if err != nil { - s.errs = append(s.errs, errors.Wrapf(err, "ParseValue[%v]: %v", i, run.Http.Url)) + s.errs = append(s.errs, errors.Wrapf(err, "ParseValue[%v]: %v", i, run.HTTP.URL)) continue } @@ -239,8 +245,8 @@ func doScenario(in *doScenarioInput) error { } e := httpexpect.New(s, u.Scheme+"://"+u.Host) - req := e.Request(run.Http.Method, u.Path) - for k, v := range run.Http.Headers { + req := e.Request(run.HTTP.Method, u.Path) + for k, v := range run.HTTP.Headers { fn := fmt.Sprintf("%v_hdr.%v", prefix, k) nv, err := s.ParseValue(v, fn) if err != nil { @@ -252,48 +258,48 @@ func doScenario(in *doScenarioInput) error { log.Printf("[header] %v: %v", k, nv) } - for k, v := range run.Http.QueryParams { + for k, v := range run.HTTP.QueryParams { fn := fmt.Sprintf("%v_qparams.%v", prefix, k) nv, _ := s.ParseValue(v, fn) req = req.WithQuery(k, nv) } - for k, v := range run.Http.Forms { + for k, v := range run.HTTP.Forms { fn := fmt.Sprintf("%v_forms.%v", prefix, k) nv, _ := s.ParseValue(v, fn) req = req.WithFormField(k, nv) } - if run.Http.Payload != "" { + if run.HTTP.Payload != "" { fn := fmt.Sprintf("%v_payload", prefix) - nv, _ := s.ParseValue(run.Http.Payload, fn) + nv, _ := s.ParseValue(run.HTTP.Payload, fn) req = req.WithBytes([]byte(nv)) } resp := req.Expect() - if run.Http.ResponseOut != "" { + if run.HTTP.ResponseOut != "" { body := resp.Body().Raw() - s.Write(run.Http.ResponseOut, []byte(body)) + s.Write(run.HTTP.ResponseOut, []byte(body)) log.Printf("[response] %v", body) } - if run.Http.Asserts == nil { + if run.HTTP.Asserts == nil { continue } - resp = resp.Status(run.Http.Asserts.Code) + resp = resp.Status(run.HTTP.Asserts.Code) - if run.Http.Asserts.ValidateJSON != "" { - resp.JSON().Schema(run.Http.Asserts.ValidateJSON) + if run.HTTP.Asserts.ValidateJSON != "" { + resp.JSON().Schema(run.HTTP.Asserts.ValidateJSON) } - if run.Http.Asserts.Script != "" { + if run.HTTP.Asserts.Script != "" { fn := fmt.Sprintf("%v_assertscript", prefix) - s.WriteScript(fn, run.Http.Asserts.Script) + s.WriteScript(fn, run.HTTP.Asserts.Script) b, err := s.RunScript(fn) if err != nil { s.errs = append(s.errs, errors.Wrapf(err, - "assert.script[%v]:\n%v: %v", i, run.Http.Asserts.Script, string(b))) + "assert.script[%v]:\n%v: %v", i, run.HTTP.Asserts.Script, string(b))) } else { if len(string(b)) > 0 { log.Printf("asserts.script[%v]:\n%v", i, string(b)) From 7d1ee4b60e828557c614e254fe2f38daa33dd1ad Mon Sep 17 00:00:00 2001 From: Syakir Date: Wed, 3 Feb 2021 02:10:58 +0000 Subject: [PATCH 2/7] feat: Add support for file upload --- scenario.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/scenario.go b/scenario.go index dfa1712..b6b1c88 100644 --- a/scenario.go +++ b/scenario.go @@ -264,6 +264,15 @@ func doScenario(in *doScenarioInput) error { req = req.WithQuery(k, nv) } + if len(run.HTTP.Files) > 0 { + req = req.WithMultipart() + } + for k, v := range run.HTTP.Files { + fn := fmt.Sprintf("%v_files.%v", prefix, k) + nv, _ := s.ParseValue(v, fn) + req = req.WithFile(k, nv) + } + for k, v := range run.HTTP.Forms { fn := fmt.Sprintf("%v_forms.%v", prefix, k) nv, _ := s.ParseValue(v, fn) From 626abf07d8009c22b5eeacdc3f17329058891f26 Mon Sep 17 00:00:00 2001 From: Syakir Date: Wed, 3 Feb 2021 02:15:29 +0000 Subject: [PATCH 3/7] feat: Check for wrong directory --- main.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/main.go b/main.go index 2ec5aba..b9f0950 100644 --- a/main.go +++ b/main.go @@ -101,6 +101,10 @@ func combineFilesAndDir() []string { final = append(final, k) } + if len(final) == 0 { + log.Fatal("No files found. Please recheck directory.") + } + return final } From ed5923bc685770c13711e9ccc80edb18034f3f82 Mon Sep 17 00:00:00 2001 From: Syakir Date: Wed, 3 Feb 2021 02:17:45 +0000 Subject: [PATCH 4/7] feat: Error check for filepath walk --- main.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/main.go b/main.go index b9f0950..7c3ea81 100644 --- a/main.go +++ b/main.go @@ -82,7 +82,7 @@ func combineFilesAndDir() []string { tmp[f] = struct{}{} } - filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + walkErr := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { if err != nil { return err } @@ -95,6 +95,9 @@ func combineFilesAndDir() []string { return nil }) + if walkErr != nil { + log.Fatalf("Filepath walk failed: %v", walkErr) + } var final []string for k, _ := range tmp { From d5a7af53cec97818ce4a9ef1b75a2f5826ee6e42 Mon Sep 17 00:00:00 2001 From: Syakir Date: Wed, 3 Feb 2021 02:24:50 +0000 Subject: [PATCH 5/7] fix: Linting --- README.md | 140 +++++++++++++++++++++++++++++------------------------- main.go | 11 +++-- slack.go | 7 ++- 3 files changed, 86 insertions(+), 72 deletions(-) diff --git a/README.md b/README.md index 3afe4a7..f14885d 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,15 @@ +# OOPS + ![main](https://github.com/flowerinthenight/oops/workflows/main/badge.svg) ## Overview + `oops` is an automation-friendly, highly-scalable, and scriptable API/generic testing tool built to run on [Kubernetes](https://kubernetes.io/). It accepts and runs file-based test cases, or 'scenarios' written in [YAML](https://yaml.org/). ## Running in a local environment + You can install the binary and run local scenario file(s). This is useful for testing your scenario file(s) before deployment. + ```bash # Install the binary. $ brew tap flowerinthenight/tap @@ -23,6 +28,7 @@ $ oops --dir ./examples/ ``` ## Deploying to Kubernetes + To scale the testing workload, this tool will attempt to distribute all scenario files to all worker pods using pub/sub messaging (currently supports SNS+SQS, and GCP PubSub). At the moment, it needs to be triggered first before the actual execution starts. The trigger payload is `{"code":"start"}`. It is recommended that your scenario files are isolated at the file level. That means that as much as possible, a single scenario file is standalone. For integration tests, a single scenario file could contain multiple related test cases (i.e. create, inspect, delete type of tests) in it. @@ -32,20 +38,22 @@ Although this tool was built to run on k8s, it will work just fine in any enviro An example [`deployment.yaml`](https://github.com/flowerinthenight/oops/blob/master/deployment.yaml) for k8s using GCP PubSub is provided for reference. Make sure to update the relevant values for your own setup. ## Scenario file + The following is the specification of a valid scenario file. All scenario files must have a `.yaml` extension. + ```yaml tags: # Tag(s) for this scenario file. If these tag combinations match with what is # provided in the --tags flag (if any), this scenario file is allowed to run. key1: value1 key2: value2 - + env: # These key/values will be added to the environment variables in your local # or in the pod oops will be running on. ENV_KEY1: value1 ENV_KEY2: value2 - + # Any value that starts with '#!' (i.e. #!/bin/bash) will be written to disk as # an executable script file and the resulting output combined from stdout & stderr # will become the final evaluated value. This is useful if you chain http calls, @@ -68,69 +76,69 @@ prepare: | # A list of http requests to perform sequentially. This tool will continue running # all the list entries even if failure occurs during the execution. run: -- http: - method: POST - - # Filename: /.yaml_run_url - # Example: /tmp/scenario01.yaml_run0_url - url: "https://service.alphaus.cloud/users" - - # Filename: /.yaml_run_hdr. - # Example: /tmp/scenario01.yaml_run0_hdr.Authorization - headers: - Authorization: | - #!/bin/bash - echo -n "Bearer $TOKEN" - Content-Type: application/json - - # Filename: /.yaml_run_qparams. - # Example: /tmp/scenario01.yaml_run0_qparams.key2 - query_params: - key1: value1 - key2: | - #!/bin/bash - echo -n "$KEY2" - - # Filename: /.yaml_run_forms. - # Example: /tmp/scenario01.yaml_run0_forms.key2 - forms: - key1: value1 - key2: | - #!/bin/bash - echo -n "$KEY2" - - # Filename: /.yaml_run_payload - # Example: /tmp/scenario01.yaml_run0_payload - payload: | - {"key1":"value1","key2":"value2"} - - # If response payload is not empty, its contents will be written in the - # file below. Useful if you want to refer to it under 'asserts.script' - # and/or 'check'. - response_out: /tmp/out.json - - asserts: - # The expected http status code. Indicates a failure if not equal. - status_code: 200 - - # JSON validation using https://github.com/xeipuuv/gojsonschema package. - validate_json: | - { - "type": "object", - "properties": { - ... + - http: + method: POST + + # Filename: /.yaml_run_url + # Example: /tmp/scenario01.yaml_run0_url + url: "https://service.alphaus.cloud/users" + + # Filename: /.yaml_run_hdr. + # Example: /tmp/scenario01.yaml_run0_hdr.Authorization + headers: + Authorization: | + #!/bin/bash + echo -n "Bearer $TOKEN" + Content-Type: application/json + + # Filename: /.yaml_run_qparams. + # Example: /tmp/scenario01.yaml_run0_qparams.key2 + query_params: + key1: value1 + key2: | + #!/bin/bash + echo -n "$KEY2" + + # Filename: /.yaml_run_forms. + # Example: /tmp/scenario01.yaml_run0_forms.key2 + forms: + key1: value1 + key2: | + #!/bin/bash + echo -n "$KEY2" + + # Filename: /.yaml_run_payload + # Example: /tmp/scenario01.yaml_run0_payload + payload: | + {"key1":"value1","key2":"value2"} + + # If response payload is not empty, its contents will be written in the + # file below. Useful if you want to refer to it under 'asserts.script' + # and/or 'check'. + response_out: /tmp/out.json + + asserts: + # The expected http status code. Indicates a failure if not equal. + status_code: 200 + + # JSON validation using https://github.com/xeipuuv/gojsonschema package. + validate_json: | + { + "type": "object", + "properties": { + ... + } } - } - - # A non-zero return value indicates a failure. - # Filename: /.yaml_run_assertscript - # Example: /tmp/scenario01.yaml_run0_assertscript - script: | - #!/bin/bash - if [[ "$(cat /tmp/out.json | jq -r .username)" != "user01" ]]; then - echo "try fail" - exit 1 - fi + + # A non-zero return value indicates a failure. + # Filename: /.yaml_run_assertscript + # Example: /tmp/scenario01.yaml_run0_assertscript + script: | + #!/bin/bash + if [[ "$(cat /tmp/out.json | jq -r .username)" != "user01" ]]; then + echo "try fail" + exit 1 + fi # A script to run after 'run', if present. Useful also as a standalone script # in itself, if 'run' is empty. A non-zero return value indicates a failure. @@ -143,10 +151,12 @@ check: | Example [scenario files](https://github.com/flowerinthenight/oops/tree/master/examples) are provided for reference as well. You can run them as is. ----- +--- ## TODO + PR's are welcome! + - [ ] Parsing and assertions for response JSON payloads - [x] Labels/tags for filtering what tests to run - [x] Support for other scripting engines other than `bash/sh`, i.e. Python diff --git a/main.go b/main.go index 7c3ea81..61bc4c9 100644 --- a/main.go +++ b/main.go @@ -60,7 +60,7 @@ type cmd struct { // To identify a batch. Sent by the initiator together with // the 'process' code. - Id string `json:"id"` + ID string `json:"id"` // The file to process. Sent together with the 'process' code. Scenario string `json:"scenario"` @@ -100,7 +100,7 @@ func combineFilesAndDir() []string { } var final []string - for k, _ := range tmp { + for k := range tmp { final = append(final, k) } @@ -117,7 +117,7 @@ func distributePubsub(app *appctx) { for _, f := range final { nc := cmd{ Code: "process", - Id: id, + ID: id, Scenario: f, } @@ -148,7 +148,7 @@ func distributeSQS(app *appctx) { for _, f := range final { nc := cmd{ Code: "process", - Id: id, + ID: id, Scenario: f, } @@ -256,7 +256,8 @@ func run(ctx context.Context, done chan error) { } app := &appctx{mtx: &sync.Mutex{}} - ctx0, _ := context.WithCancel(ctx) + ctx0, cancelCtx0 := context.WithCancel(ctx) + defer cancelCtx0() done0 := make(chan error, 1) switch { diff --git a/slack.go b/slack.go index 1095f85..256d12f 100644 --- a/slack.go +++ b/slack.go @@ -6,6 +6,7 @@ import ( "net/http" ) +//SlackAttachment represents Slack Attachment structure for Slack API type SlackAttachment struct { // Fallback is our simple fallback text equivalent. Fallback string `json:"fallback"` @@ -36,17 +37,19 @@ type SlackAttachment struct { Timestamp int64 `json:"ts,omitempty"` } +//SlackMessage represents Slack message structure for Slack API type SlackMessage struct { Attachments []SlackAttachment `json:"attachments"` } -func (sn *SlackMessage) Notify(slackUrl string) error { +//Notify post the message via Slack API +func (sn *SlackMessage) Notify(slackURL string) error { bp, err := json.Marshal(sn) if err != nil { return err } - _, err = http.Post(slackUrl, "application/json", bytes.NewBuffer(bp)) + _, err = http.Post(slackURL, "application/json", bytes.NewBuffer(bp)) if err != nil { return err } From f8f025ce984091dbd5ff38f46dde641014ab76ec Mon Sep 17 00:00:00 2001 From: Syakir Date: Wed, 3 Feb 2021 02:33:41 +0000 Subject: [PATCH 6/7] feat: Support for maintainers --- scenario.go | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/scenario.go b/scenario.go index b6b1c88..312c05c 100644 --- a/scenario.go +++ b/scenario.go @@ -53,11 +53,12 @@ type ReportPubsub struct { // Scenario represents a single scenario file to run. type Scenario struct { - Tags map[string]string `yaml:"tags"` - Env map[string]string `yaml:"env"` - Prepare string `yaml:"prepare"` - Run []Run `yaml:"run"` - Check string `yaml:"check"` + Maintainers []string `yaml:"maintainers"` + Tags map[string]string `yaml:"tags"` + Env map[string]string `yaml:"env"` + Prepare string `yaml:"prepare"` + Run []Run `yaml:"run"` + Check string `yaml:"check"` me *Scenario input *doScenarioInput @@ -344,7 +345,7 @@ func doScenario(in *doScenarioInput) error { { Color: "danger", Title: fmt.Sprintf("%v - failure", filepath.Base(f)), - Text: fmt.Sprintf("%v", s.errs), + Text: fmt.Sprintf("Maintainers: %v\n%v", strings.Join(s.Maintainers, ", "), s.errs), Footer: "oops", Timestamp: time.Now().Unix(), }, From 59a1bbcec88098cf53ff6616056db19ca256a736 Mon Sep 17 00:00:00 2001 From: Syakir Date: Wed, 3 Feb 2021 03:55:11 +0000 Subject: [PATCH 7/7] fix: File checking --- main.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/main.go b/main.go index 61bc4c9..2fe0b5b 100644 --- a/main.go +++ b/main.go @@ -82,7 +82,7 @@ func combineFilesAndDir() []string { tmp[f] = struct{}{} } - walkErr := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { if err != nil { return err } @@ -95,13 +95,15 @@ func combineFilesAndDir() []string { return nil }) - if walkErr != nil { - log.Fatalf("Filepath walk failed: %v", walkErr) - } var final []string for k := range tmp { - final = append(final, k) + _, err := os.Stat(k) + if os.IsNotExist(err) { + log.Printf("File does not exist: %v", k) + } else { + final = append(final, k) + } } if len(final) == 0 {