Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add Percentile support #86

Closed
wants to merge 5 commits into from
Closed

Add Percentile support #86

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
e165ab9
Add Percentile support
zhiburt Oct 7, 2022
2cb7be8
Merge scenarius when doing p(95)
zhiburt Oct 7, 2022
da6968d
Merge branch 'master' of github.com:ddosify/ddosify into patch-percen…
zhiburt Oct 18, 2022
97cbdd4
Report percentile only on report stage + add CLI flags
zhiburt Oct 18, 2022
6a39bcd
Fix ordering of percentiles in stdout
zhiburt Oct 18, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 9 additions & 7 deletions config/json.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,13 +98,14 @@ func (s *step) UnmarshalJSON(data []byte) error {
}

type JsonReader struct {
ReqCount int `json:"request_count"`
LoadType string `json:"load_type"`
Duration int `json:"duration"`
TimeRunCount timeRunCount `json:"manual_load"`
Steps []step `json:"steps"`
Output string `json:"output"`
Proxy string `json:"proxy"`
ReqCount int `json:"request_count"`
LoadType string `json:"load_type"`
Duration int `json:"duration"`
TimeRunCount timeRunCount `json:"manual_load"`
Steps []step `json:"steps"`
Output string `json:"output"`
Proxy string `json:"proxy"`
OutputPercentile bool `json:"output_percentile"`
}

func (j *JsonReader) UnmarshalJSON(data []byte) error {
Expand Down Expand Up @@ -182,6 +183,7 @@ func (j *JsonReader) CreateHammer() (h types.Hammer, err error) {
Scenario: s,
Proxy: p,
ReportDestination: j.Output,
ReportPercentiles: j.OutputPercentile,
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Some places use "Percentile" others use "Percentiles". I think we can use "Percentile" everywhere.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Or "Percentiles". Whatever you want, but let's make the naming consistent

}
return
}
Expand Down
2 changes: 1 addition & 1 deletion core/engine.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,7 @@ func (e *engine) Init() (err error) {
return
}

if err = e.reportService.Init(); err != nil {
if err = e.reportService.Init(e.hammer.ReportPercentiles); err != nil {
return
}

Expand Down
51 changes: 45 additions & 6 deletions core/report/aggregator.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@
package report

import (
"math"
"sort"
"strings"
"time"

Expand All @@ -39,6 +41,7 @@ func aggregate(result *Result, response *types.Response) {
StatusCodeDist: make(map[int]int, 0),
ErrorDist: make(map[string]int),
Durations: map[string]float32{},
TotalDurations: map[string][]float32{},
}
}
item := result.ItemReports[rr.ScenarioItemID]
Expand All @@ -60,7 +63,12 @@ func aggregate(result *Result, response *types.Response) {
}
}
}
}

for _, report := range result.ItemReports {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We don't need this extra loop.

Copy link
Member

@kursataktas kursataktas Oct 21, 2022 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We can add below code snippet at line 57 or anywhere in that else statement

if percentileReportEnabled {
   item.TotalDurations["duration"] = append(item.TotalDurations["duration"], float32(rr.Duration.Seconds())) 
}

I think the aggregator should store durations only if the percentage report is enabled.

for key, duration := range report.Durations {
report.TotalDurations[key] = append(report.TotalDurations[key], duration)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We don't need to store other durations (DNS, TLS, etc.) total duration would be enough ("duration") for now.

}
}

// Don't change avg duration if there is a error
Expand Down Expand Up @@ -96,12 +104,29 @@ func (r *Result) failedPercentage() int {
}

type ScenarioItemReport struct {
Name string `json:"name"`
StatusCodeDist map[int]int `json:"status_code_dist"`
ErrorDist map[string]int `json:"error_dist"`
Durations map[string]float32 `json:"durations"`
SuccessCount int64 `json:"success_count"`
FailedCount int64 `json:"fail_count"`
Name string `json:"name"`
StatusCodeDist map[int]int `json:"status_code_dist"`
ErrorDist map[string]int `json:"error_dist"`
Durations map[string]float32 `json:"durations"`
TotalDurations map[string][]float32 `json:"total_durations"`
SuccessCount int64 `json:"success_count"`
FailedCount int64 `json:"fail_count"`
}

func (s *ScenarioItemReport) DurationPercentile(p int) float32 {
if p < 0 || p > 100 {
return 0
}

durations, ok := s.TotalDurations["duration"]
if !ok {
return 0
}

// todo: it could be optimized by always sorted array being used in TotalDurations so we would not make this call.
sort.Slice(durations, func(i, j int) bool { return durations[i] < durations[j] })

return Percentile(durations, p)
}

func (s *ScenarioItemReport) successPercentage() int {
Expand All @@ -118,3 +143,17 @@ func (s *ScenarioItemReport) failedPercentage() int {
}
return 100 - s.successPercentage()
}

func Percentile(list []float32, p int) float32 {
if p < 0 || p > 100 {
return 0
}

n := int(math.Round((float64(p) / 100.0) * float64(len(list))))
if n > 0 {
// I am not sure about the case where n == 0
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No need to cover this unless we need a percentile less than p50, we can remove this check.

n--
}

return list[n]
}
8 changes: 4 additions & 4 deletions core/report/base.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,18 +32,18 @@ var AvailableOutputServices = make(map[string]ReportService)
// ReportService is the interface that abstracts different report implementations.
type ReportService interface {
DoneChan() <-chan struct{}
Init() error
Init(reportPercentiles bool) error
Start(input chan *types.Response)
Report()
}

// NewReportService is the factory method of the ReportService.
func NewReportService(s string) (service ReportService, err error) {
if val, ok := AvailableOutputServices[s]; ok {
func NewReportService(output string) (service ReportService, err error) {
if val, ok := AvailableOutputServices[output]; ok {
// Create a new object from the service type
service = reflect.New(reflect.TypeOf(val).Elem()).Interface().(ReportService)
} else {
err = fmt.Errorf("unsupported output type: %s", s)
err = fmt.Errorf("unsupported output type: %s", output)
}

return
Expand Down
37 changes: 27 additions & 10 deletions core/report/stdout.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,22 +45,24 @@ func init() {
}

type stdout struct {
doneChan chan struct{}
result *Result
printTicker *time.Ticker
mu sync.Mutex
doneChan chan struct{}
result *Result
printTicker *time.Ticker
mu sync.Mutex
reportPercentiles bool
}

var blue = color.New(color.FgHiBlue).SprintFunc()
var green = color.New(color.FgHiGreen).SprintFunc()
var red = color.New(color.FgHiRed).SprintFunc()
var realTimePrintInterval = time.Duration(1500) * time.Millisecond

func (s *stdout) Init() (err error) {
func (s *stdout) Init(reportPercentiles bool) (err error) {
s.doneChan = make(chan struct{})
s.result = &Result{
ItemReports: make(map[int16]*ScenarioItemReport),
}
s.reportPercentiles = reportPercentiles

color.Cyan("%s Initializing... \n", emoji.Gear)
return
Expand Down Expand Up @@ -108,11 +110,10 @@ func (s *stdout) realTimePrintStart() {

func (s *stdout) liveResultPrint() {
fmt.Fprintf(out, "%s %s %s\n",
green(fmt.Sprintf("%s Successful Run: %-6d %3d%% %5s",
emoji.CheckMark, s.result.SuccessCount, s.result.successPercentage(), "")),
red(fmt.Sprintf("%s Failed Run: %-6d %3d%% %5s",
emoji.CrossMark, s.result.FailedCount, s.result.failedPercentage(), "")),
blue(fmt.Sprintf("%s Avg. Duration: %.5fs", emoji.Stopwatch, s.result.AvgDuration)))
green(fmt.Sprintf("%s Successful Run: %-6d %3d%% %5s", emoji.CheckMark, s.result.SuccessCount, s.result.successPercentage(), "")),
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The max. line-length is limited to 120 chars as stated in the linter settings (at .golangci.yml). Let's follow the standards.

red(fmt.Sprintf("%s Failed Run: %-6d %3d%% %5s", emoji.CrossMark, s.result.FailedCount, s.result.failedPercentage(), "")),
blue(fmt.Sprintf("%s Avg. Duration: %.5fs", emoji.Stopwatch, s.result.AvgDuration)),
)
}

func (s *stdout) realTimePrintStop() {
Expand Down Expand Up @@ -173,6 +174,22 @@ func (s *stdout) printDetails() {
fmt.Fprintf(w, " %s\t:%.4fs\n", v.name, v.duration)
}

if s.reportPercentiles {
// print percentalies
percentiles := []map[string]float32{
{"P99": v.DurationPercentile(99)},
{"P95": v.DurationPercentile(95)},
{"P90": v.DurationPercentile(90)},
{"P80": v.DurationPercentile(80)},
}
fmt.Fprintln(w, "\nPercentiles:")
for _, val := range percentiles {
for name, percentile := range val {
fmt.Fprintf(w, " %s\t:%.4fs\n", name, percentile)
}
}
}

if len(v.StatusCodeDist) > 0 {
fmt.Fprintln(w, "\nStatus Code (Message) :Count")
for s, c := range v.StatusCodeDist {
Expand Down
68 changes: 56 additions & 12 deletions core/report/stdoutJson.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,8 @@ package report

import (
"encoding/json"
"fmt"
"math"
"os"

"go.ddosify.com/ddosify/core/types"
)
Expand All @@ -35,15 +35,18 @@ func init() {
}

type stdoutJson struct {
doneChan chan struct{}
result *Result
doneChan chan struct{}
result *Result
reportPercentiles bool
}

func (s *stdoutJson) Init() (err error) {
func (s *stdoutJson) Init(reportPercentiles bool) (err error) {
s.doneChan = make(chan struct{})
s.result = &Result{
ItemReports: make(map[int16]*ScenarioItemReport),
}
s.reportPercentiles = reportPercentiles

return
}

Expand All @@ -54,12 +57,57 @@ func (s *stdoutJson) Start(input chan *types.Response) {
s.doneChan <- struct{}{}
}

type jsonResult struct {
SuccessCount int64 `json:"success_count"`
FailedCount int64 `json:"fail_count"`
AvgDuration float32 `json:"avg_duration"`
ItemReports map[int16]*jsonScenarioItemReport `json:"steps"`
}

type jsonScenarioItemReport struct {
Name string `json:"name"`
StatusCodeDist map[int]int `json:"status_code_dist"`
ErrorDist map[string]int `json:"error_dist"`
Durations map[string]float32 `json:"durations"`
Percentiles map[string]float32 `json:"percentiles"`
SuccessCount int64 `json:"success_count"`
FailedCount int64 `json:"fail_count"`
}

func (s *stdoutJson) Report() {
p := 1e3
jsonResult := jsonResult{
SuccessCount: s.result.SuccessCount,
FailedCount: s.result.FailedCount,
AvgDuration: s.result.AvgDuration,
ItemReports: map[int16]*jsonScenarioItemReport{},
}

for key, item := range s.result.ItemReports {
jsonResult.ItemReports[key] = &jsonScenarioItemReport{
Name: item.Name,
StatusCodeDist: item.StatusCodeDist,
ErrorDist: item.ErrorDist,
Durations: item.Durations,
SuccessCount: item.SuccessCount,
FailedCount: item.FailedCount,
}

if s.reportPercentiles {
Copy link
Member

@kursataktas kursataktas Oct 21, 2022 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If possible, let's not show the percentiles key if it is not enabled.

Current state:

...
 "durations": {
        "connection": 0.005,
        "dns": 0.009,
        "request_write": 0,
        "response_read": 0.382,
        "server_processing": 0.253,
        "tls": 0.012,
        "total": 0.661
      },
      "percentiles": null, ---> Let's remove this if we can
      "success_count": 10,
      "fail_count": 0
    },
...

jsonResult.ItemReports[key].Percentiles = map[string]float32{
"p99": item.DurationPercentile(99),
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Milliseconds would be enough.

Current stdout-json example:

...
"percentiles": {
        "p80": 0.12210735, --> "0.122" would be enough
        "p90": 0.14628321,
        "p95": 0.1905153,
        "p99": 0.1905153
      },
...

"p95": item.DurationPercentile(95),
"p90": item.DurationPercentile(90),
"p80": item.DurationPercentile(80),
}
}

}

p := 1e3
s.result.AvgDuration = float32(math.Round(float64(s.result.AvgDuration)*p) / p)

for _, itemReport := range s.result.ItemReports {
for _, item := range jsonResult.ItemReports {
itemReport := item
durations := make(map[string]float32)
for d, s := range itemReport.Durations {
// Less precision for durations.
Expand All @@ -69,8 +117,8 @@ func (s *stdoutJson) Report() {
itemReport.Durations = durations
}

j, _ := json.Marshal(s.result)
printJson(j)
encoder := json.NewEncoder(os.Stdout)
encoder.Encode(&jsonResult)
}

func (s *stdoutJson) DoneChan() <-chan struct{} {
Expand Down Expand Up @@ -107,10 +155,6 @@ func (s ScenarioItemReport) MarshalJSON() ([]byte, error) {
})
}

var printJson = func(j []byte) {
fmt.Println(string(j))
}

var strKeyToJsonKey = map[string]string{
"dnsDuration": "dns",
"connDuration": "connection",
Expand Down
9 changes: 7 additions & 2 deletions core/report/stdoutJson_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ package report
import (
"bytes"
"encoding/json"
"fmt"
"reflect"
"testing"
"time"
Expand All @@ -31,7 +32,7 @@ import (

func TestInitStdoutJson(t *testing.T) {
sj := &stdoutJson{}
sj.Init()
sj.Init(false)

if sj.doneChan == nil {
t.Errorf("DoneChan should be initialized")
Expand Down Expand Up @@ -130,7 +131,7 @@ func TestStdoutJsonStart(t *testing.T) {
}

s := &stdoutJson{}
s.Init()
s.Init(false)

responseChan := make(chan *types.Response, len(responses))
go s.Start(responseChan)
Expand Down Expand Up @@ -254,3 +255,7 @@ func TestStdoutJsonOutput(t *testing.T) {
t.Errorf("Expected: %v, Found: %v", expectedOutput, output)
}
}

var printJson = func(j []byte) {
fmt.Println(string(j))
}
4 changes: 2 additions & 2 deletions core/report/stdout_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,7 @@ func TestResult(t *testing.T) {

func TestInit(t *testing.T) {
s := &stdout{}
s.Init()
s.Init(false)

if s.doneChan == nil {
t.Errorf("DoneChan should be initialized")
Expand Down Expand Up @@ -190,7 +190,7 @@ func TestStart(t *testing.T) {
}

s := &stdout{}
s.Init()
s.Init(false)

responseChan := make(chan *types.Response, len(responses))
go s.Start(responseChan)
Expand Down
3 changes: 3 additions & 0 deletions core/types/hammer.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,9 @@ type Hammer struct {
// Destination of the results data.
ReportDestination string

// Report percentiles
ReportPercentiles bool

// Dynamic field for extra parameters.
Others map[string]interface{}
}
Expand Down
Loading