Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Github runner add benchmark #1599

Merged
merged 103 commits into from
Oct 25, 2024
Merged
Changes from 14 commits
Commits
Show all changes
103 commits
Select commit Hold shift + click to select a range
88ebf59
add file-to-blackhole ilogtail benchmark
Assassin718 Jul 11, 2024
762a3f3
fix lint check errors
Assassin718 Jul 11, 2024
6078a37
add file-to-blackhole performance test
Assassin718 Jul 11, 2024
9d700f5
add benchmark CI
Assassin718 Jul 20, 2024
49812f9
update benmark.yaml from using benchmark action to just cat result.json
Assassin718 Jul 21, 2024
2a455f5
add CI summary
Assassin718 Jul 23, 2024
11e7aa3
trigger benchmark CI
Assassin718 Jul 24, 2024
c545ad9
modify benmark.yaml add summary
Assassin718 Jul 24, 2024
3f3521a
update benchmark CI to only trigger on merge, update test_cases case.…
Assassin718 Jul 25, 2024
9dfd2db
add benchmark docs benchmark.md and delete a.log
Assassin718 Jul 29, 2024
439fbe1
update github benchmark CI to store avg/max ans time records
Assassin718 Aug 7, 2024
d4562e0
update trigger file.go
Assassin718 Aug 11, 2024
13204b0
update trigger file.go lint error
Assassin718 Aug 11, 2024
8e2a268
update trigger file.go
Assassin718 Aug 11, 2024
c1257b0
polish
Assassin718 Aug 19, 2024
c0505be
resolve conflict
Assassin718 Aug 26, 2024
cdbf522
resolve go.mod conflict
Assassin718 Sep 17, 2024
b67d5d8
update benchmark.md
Assassin718 Sep 19, 2024
1760f0a
update benchmark.md
Assassin718 Sep 19, 2024
05e30c4
update benchmark.yaml
Assassin718 Sep 23, 2024
0341562
resolve conflict
Assassin718 Sep 23, 2024
3d26c8d
update benchmark.yaml
Assassin718 Sep 24, 2024
e2cd619
update benchmark.yaml
Assassin718 Sep 24, 2024
c6c86d5
update benchmark.yaml
Assassin718 Sep 24, 2024
09f1467
update benmark.yam
Assassin718 Sep 24, 2024
9714ec1
test
Assassin718 Sep 29, 2024
8fb21e7
polish
Assassin718 Sep 29, 2024
73bdca4
Merge branch 'main' into e2e-performance-test
Assassin718 Oct 15, 2024
49d0a9d
update clean up and benchmark structure
Assassin718 Oct 15, 2024
66cd525
Merge branch 'e2e-performance-test' into benchmark
Assassin718 Oct 15, 2024
c8932d3
polish
Assassin718 Oct 15, 2024
0ced77f
polish
Assassin718 Oct 15, 2024
6287cdb
resolve lint
Assassin718 Oct 15, 2024
6dde18f
Merge branch 'main' into e2e-performance-test
Assassin718 Oct 15, 2024
8b2005b
Merge branch 'main' into benchmark
Assassin718 Oct 15, 2024
6e61c0f
update benchmark.yaml
Assassin718 Oct 15, 2024
087e497
fix bug
Assassin718 Oct 15, 2024
e7d3461
update benchmark ilogtail
Assassin718 Oct 16, 2024
c683b51
Merge branch 'e2e-performance-test' into benchmark
Assassin718 Oct 16, 2024
9bcc8ad
update benchmark.yaml
Assassin718 Oct 16, 2024
d586689
update Makefile
Assassin718 Oct 16, 2024
115580d
polish
Assassin718 Oct 16, 2024
0a7a5ef
fix bug
Assassin718 Oct 16, 2024
6402c1d
polish
Assassin718 Oct 16, 2024
bf80fd6
polish
Assassin718 Oct 16, 2024
90a8a38
Merge branch 'benchmark' into e2e-performance-test
Assassin718 Oct 16, 2024
4f44830
increase log generate speed
Assassin718 Oct 16, 2024
4fe6249
test
Assassin718 Oct 16, 2024
311837d
update ilogtail docker-compose.yaml
Assassin718 Oct 16, 2024
1407f60
Merge branch 'benchmark' into e2e-performance-test
Assassin718 Oct 16, 2024
0d9dfe2
polish
Assassin718 Oct 16, 2024
34d3f3f
update doc
Assassin718 Oct 17, 2024
506f811
merge to e2e-performance-test
Assassin718 Oct 17, 2024
70a422d
test benchmark action with write permission
Assassin718 Oct 18, 2024
9fdacf5
fix bug
Assassin718 Oct 18, 2024
b3c10cd
update log generator and monitor
Assassin718 Oct 18, 2024
b07d0fa
resolve lint
Assassin718 Oct 18, 2024
546be74
empty commit
Assassin718 Oct 18, 2024
267e012
update monitor
Assassin718 Oct 18, 2024
15815d0
empty commit
Assassin718 Oct 18, 2024
7429e2b
update monitor
Assassin718 Oct 18, 2024
93686e3
update monitor quit logic and file trigger logic
Assassin718 Oct 18, 2024
c69d538
update file trigger
Assassin718 Oct 18, 2024
99f0582
update case.feature
Assassin718 Oct 18, 2024
c47a2e1
resolve lint
Assassin718 Oct 18, 2024
eedeb7a
fix lint
Assassin718 Oct 18, 2024
e2df97b
fix lint
Assassin718 Oct 18, 2024
b55203c
update file trigger
Assassin718 Oct 18, 2024
5427525
update file trigger
Assassin718 Oct 18, 2024
78e2165
update feature
Assassin718 Oct 18, 2024
86e2ad1
update case.feature
Assassin718 Oct 18, 2024
0afc9ec
update benchmark.md
Assassin718 Oct 21, 2024
e509874
update trigger to nginx logs
Assassin718 Oct 21, 2024
4a8d26a
update trigger to nginx logs
Assassin718 Oct 21, 2024
2bdf8fe
polish
Assassin718 Oct 21, 2024
0cd228f
polish
Assassin718 Oct 21, 2024
3623c97
polish
Assassin718 Oct 21, 2024
94e5e5b
update ilogtailspl
Assassin718 Oct 21, 2024
a75e538
empty commit
Assassin718 Oct 22, 2024
d938bc2
fix bug
Assassin718 Oct 22, 2024
62ce72f
update spl ilogtail.yaml
Assassin718 Oct 22, 2024
c16c262
update docs
Assassin718 Oct 22, 2024
cdb2c26
update github action benchmark.yaml
Assassin718 Oct 22, 2024
3141b11
update trigger file
Assassin718 Oct 23, 2024
dc9c392
add testcases filebeat, logstash
Assassin718 Oct 23, 2024
42d42ab
polish
Assassin718 Oct 23, 2024
26374cf
Merge branch 'main' into e2e-performance-test
Assassin718 Oct 23, 2024
000f3a7
resolve conflict
Assassin718 Oct 24, 2024
c7b7a0c
Merge branch 'main' into benchmark
Assassin718 Oct 24, 2024
62aaf06
update monitor quit logic
Assassin718 Oct 24, 2024
701483c
rm logstash benchmark case
Assassin718 Oct 24, 2024
5632bdd
resolve github benchmark bug
Assassin718 Oct 24, 2024
4f41d94
update monitor stop logic
Assassin718 Oct 24, 2024
59b556c
update monitor.go
Assassin718 Oct 24, 2024
5bac0d0
update monitor.go
Assassin718 Oct 24, 2024
4082473
update monitor.go
Assassin718 Oct 24, 2024
d9b9381
update github-action benchmark.yaml
Assassin718 Oct 24, 2024
5103da0
update monitor.go
Assassin718 Oct 24, 2024
229993f
update benchmark.yaml
Assassin718 Oct 24, 2024
b8269f4
update monitor.go
Assassin718 Oct 25, 2024
3793bf9
fix license
Assassin718 Oct 25, 2024
9403887
resolve conflict
Assassin718 Oct 25, 2024
413b6f7
resolve lint
Assassin718 Oct 25, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 3 additions & 12 deletions .github/workflows/benchmark.yaml
Original file line number Diff line number Diff line change
@@ -28,9 +28,6 @@ on:
- 'CHANGELOG.md'
# types:
# - closed
push:
branches:
- benchmark

jobs:
CI:
@@ -42,6 +39,8 @@ jobs:
go-version: [ 1.19.10 ]
runner: [ ubuntu-latest ]
fail-fast: true
permissions:
contents: write
steps:
# Clean up space to prevent action from running out of disk space.
- name: clean
@@ -85,19 +84,12 @@ jobs:
WITHOUTGDB: ON
run: make benchmark

# - name: Download previous benchmark data
# uses: actions/cache@v4
# with:
# path: ./cache
# key: ${{ runner.os }}-benchmark

- name: Store benchmark result iLogtail
uses: benchmark-action/github-action-benchmark@v1
with:
name: benchmark
tool: "customSmallerIsBetter"
output-file-path: "test/benchmark/report/ilogtail_statistic_all.json"
# external-data-json-path: ./cache/ilogtail_statistic.json
auto-push: false
summary-always: true

@@ -112,11 +104,10 @@ jobs:

# Push gh-pages branch by yourself
- name: Push benchmark result
# run: git push 'https://alibaba:${{ secrets.GITHUB_TOKEN }}@github.com/alibaba/ilogtail.git' gh-pages:gh-pages
run: git push 'https://Assassin718:${{ secrets.GITHUB_TOKEN }}@github.com/Assassin718/ilogtail.git' gh-pages:gh-pages

result:
if: github.event.pull_request.merged == true
# if: github.event.pull_request.merged == true
runs-on: ubuntu-latest
timeout-minutes: 60
needs: [ CI ]
Original file line number Diff line number Diff line change
@@ -8,5 +8,12 @@ Feature: performance file to blackhole filebeat
Given docker-compose boot type {benchmark}
When start docker-compose {performance_file_to_blackhole_filebeat}
When start monitor {filebeat}
<<<<<<< HEAD
When generate logs to file, speed {100}MB/s, total {1}min, to file {./a.log}, template
"""
{"url": "POST /PutData?Category=YunOsAccountOpLog HTTP/1.1", "ip": "10.200.98.220", "user-agent": "aliyun-sdk-java", "request": {"status": "200", "latency": "18204"}, "time": "07/Jul/2022:10:30:28"}
"""
=======
When generate random json logs to file, speed {10}MB/s, total {1}min, to file {./a.log}
When wait monitor until log processing finished
>>>>>>> benchmark
Original file line number Diff line number Diff line change
@@ -8,5 +8,12 @@ Feature: performance file to blackhole fluentbit
Given docker-compose boot type {benchmark}
When start docker-compose {performance_file_to_blackhole_fluentbit}
When start monitor {fluent-bit}
<<<<<<< HEAD
When generate logs to file, speed {100}MB/s, total {1}min, to file {./a.log}, template
"""
{"url": "POST /PutData?Category=YunOsAccountOpLog HTTP/1.1", "ip": "10.200.98.220", "user-agent": "aliyun-sdk-java", "request": {"status": "200", "latency": "18204"}, "time": "07/Jul/2022:10:30:28"}
"""
=======
When generate random json logs to file, speed {10}MB/s, total {1}min, to file {./a.log}
When wait monitor until log processing finished
>>>>>>> benchmark
Original file line number Diff line number Diff line change
@@ -8,5 +8,12 @@ Feature: performance file to blackhole iLogtail
Given docker-compose boot type {benchmark}
When start docker-compose {performance_file_to_blackhole_ilogtail}
When start monitor {ilogtailC}
<<<<<<< HEAD
When generate logs to file, speed {100}MB/s, total {1}min, to file {./a.log}, template
"""
Assassin718 marked this conversation as resolved.
Show resolved Hide resolved
{"url": "POST /PutData?Category=YunOsAccountOpLog HTTP/1.1", "ip": "10.200.98.220", "user-agent": "aliyun-sdk-java", "request": {"status": "200", "latency": "18204"}, "time": "07/Jul/2022:10:30:28"}
"""
=======
When generate random json logs to file, speed {10}MB/s, total {1}min, to file {./a.log}
When wait monitor until log processing finished
When wait monitor until log processing finished
>>>>>>> benchmark
Original file line number Diff line number Diff line change
@@ -8,5 +8,12 @@ Feature: performance file to blackhole vector
Given docker-compose boot type {benchmark}
When start docker-compose {performance_file_to_blackhole_vector}
When start monitor {vector}
<<<<<<< HEAD
When generate logs to file, speed {100}MB/s, total {1}min, to file {./a.log}, template
"""
{"url": "POST /PutData?Category=YunOsAccountOpLog HTTP/1.1", "ip": "10.200.98.220", "user-agent": "aliyun-sdk-java", "request": {"status": "200", "latency": "18204"}, "time": "07/Jul/2022:10:30:28"}
"""
=======
When generate random json logs to file, speed {10}MB/s, total {1}min, to file {./a.log}
When wait monitor until log processing finished
>>>>>>> benchmark
91 changes: 38 additions & 53 deletions test/engine/trigger/file.go
Original file line number Diff line number Diff line change
@@ -6,7 +6,6 @@ import (
"math/rand"
"os"
"path/filepath"
"sync"
"time"

"golang.org/x/time/rate"
@@ -16,64 +15,50 @@ import (

// JSON template
func GenerateRandomJSONLogToFile(ctx context.Context, speed, totalTime int, path string) (context.Context, error) {
numGoRoutines := (speed-1)/5 + 1
if numGoRoutines > 8 {
numGoRoutines = 8
JSONTemplates := []string{
Copy link
Collaborator

@messixukejia messixukejia Oct 21, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

这个内置的几个优先组合的,还是有点问题。

1、需要做成变量,更接近模拟真实的场景,例如ip是20个固定的地址,stats 200/400/401/403/404等(200多,其他的少),user-agent:aliyun-sdk-java/aliyun-sdk-golang/aliyun-sdk-python, latency:一段范围内的正态分布随机数,时间 当前日志时间。然后去正交组合。
2、把例子改成标准的Nginx日志,先正则解析。
3、增加一个logtail-spl的对比。

`{"url": "POST /PutData?Category=paskdnkwja HTTP/1.1", "ip": "10.200.98.220", "user-agent": "aliyun-sdk-java", "request": {"status": "200", "latency": "18"}, "time": "12/Sep/2024:11:30:02"}\n`,
`{"url": "GET /PutData?Category=dwds HTTP/1.1", "ip": "10.7.159.1", "user-agent": "aliyun-sdk-java", "request": {"status": "404", "latency": "22024"}, "time": "06/Jun/2001:09:35:59"}\n`,
`{"url": "GET /PutData?Category=ubkjbkjgiuiu HTTP/1.1", "ip": "172.130.98.250", "user-agent": "aliyun-sdk-java", "request": {"301": "200", "latency": "12334"}, "time": "08/Aug/2018:10:30:28"}\n`,
`{"url": "POST /PutData?Category=oasjdwbjkbjkg HTTP/1.1", "ip": "1.20.9.220", "user-agent": "aliyun-sdk-java", "request": {"401": "200", "latency": "112"}, "time": "09/Sep/2024:15:25:28"}\n`,
`{"url": "POST /PutData?Category=asdjhoiasjdoOpLog HTTP/1.1", "ip": "172.168.0.1", "user-agent": "aliyun-sdk-java", "request": {"status": "200", "latency": "8815"}, "time": "01/Jan/2022:10:30:28"}\n`,
}
maxLen := 0
for j := 0; j < len(JSONTemplates); j++ {
if len(JSONTemplates[j]) > maxLen {
maxLen = len(JSONTemplates[j])
}
}
wg := sync.WaitGroup{}
for i := 0; i < numGoRoutines; i++ {
wg.Add(1)
go func(ctx context.Context, thread_id, speed, totalTime int, path string) {
defer wg.Done()
JSONTemplates := []string{
`{"url": "POST /PutData?Category=paskdnkwja HTTP/1.1", "ip": "10.200.98.220", "user-agent": "aliyun-sdk-java", "request": {"status": "200", "latency": "18"}, "time": "12/Sep/2024:11:30:02"}\n`,
`{"url": "GET /PutData?Category=dwds HTTP/1.1", "ip": "10.7.159.1", "user-agent": "aliyun-sdk-java", "request": {"status": "404", "latency": "22024"}, "time": "06/Jun/2001:09:35:59"}\n`,
`{"url": "GET /PutData?Category=ubkjbkjgiuiu HTTP/1.1", "ip": "172.130.98.250", "user-agent": "aliyun-sdk-java", "request": {"301": "200", "latency": "12334"}, "time": "08/Aug/2018:10:30:28"}\n`,
`{"url": "POST /PutData?Category=oasjdwbjkbjkg HTTP/1.1", "ip": "1.20.9.220", "user-agent": "aliyun-sdk-java", "request": {"401": "200", "latency": "112"}, "time": "09/Sep/2024:15:25:28"}\n`,
`{"url": "POST /PutData?Category=asdjhoiasjdoOpLog HTTP/1.1", "ip": "172.168.0.1", "user-agent": "aliyun-sdk-java", "request": {"status": "200", "latency": "8815"}, "time": "01/Jan/2022:10:30:28"}\n`,
}
maxLen := 0
for j := 0; j < len(JSONTemplates); j++ {
if len(JSONTemplates[j]) > maxLen {
maxLen = len(JSONTemplates[j])
}
}

// clear file
path = filepath.Clean(path)
path = fmt.Sprintf("%d_%s", thread_id, path)
path = filepath.Join(config.CaseHome, path)
fmt.Println(path)
_ = os.WriteFile(path, []byte{}, 0600)
file, _ := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600) // #nosec G304
// clear file
path = filepath.Clean(path)
path = filepath.Join(config.CaseHome, path)
fmt.Println(path)
_ = os.WriteFile(path, []byte{}, 0600)
file, _ := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600) // #nosec G304

limiter := rate.NewLimiter(rate.Limit(speed*1024*1024), maxLen)
limiter := rate.NewLimiter(rate.Limit(speed*1024*1024), maxLen)

timeout := time.After(time.Minute * time.Duration(totalTime))
timeout := time.After(time.Minute * time.Duration(totalTime))

rand.Seed(time.Now().UnixNano())
rand.Seed(time.Now().UnixNano())

for {
select {
// context is done
case <-ctx.Done():
// clear file
_ = file.Close()
return
// all time is done
case <-timeout:
// clear file
_ = file.Close()
return
default:
if limiter.AllowN(time.Now(), maxLen) {
randomIndex := rand.Intn(len(JSONTemplates)) // #nosec G404
_, _ = file.WriteString(JSONTemplates[randomIndex])
}
}
for {
select {
// context is done
case <-ctx.Done():
// clear file
_ = file.Close()
return ctx, nil
// all time is done
case <-timeout:
// clear file
_ = file.Close()
return ctx, nil
default:
if limiter.AllowN(time.Now(), maxLen) {
randomIndex := rand.Intn(len(JSONTemplates)) // #nosec G404
_, _ = file.WriteString(JSONTemplates[randomIndex])
}
}(ctx, i, speed/numGoRoutines, totalTime, path)
}
}
wg.Wait()
return ctx, nil
}