-
Notifications
You must be signed in to change notification settings - Fork 397
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
1 parent
8493e0c
commit 05864b7
Showing
35 changed files
with
1,243 additions
and
56 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,118 @@ | ||
| # Copyright 2021 iLogtail Authors | ||
| # | ||
| # Licensed under the Apache License, Version 2.0 (the "License"); | ||
| # you may not use this file except in compliance with the License. | ||
| # You may obtain a copy of the License at | ||
| # | ||
| # http://www.apache.org/licenses/LICENSE-2.0 | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, software | ||
| # distributed under the License is distributed on an "AS IS" BASIS, | ||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| # See the License for the specific language governing permissions and | ||
| # limitations under the License. | ||
|
|
||
| name: Benchmark | ||
|
|
||
| on: | ||
| pull_request: | ||
| branches: | ||
| - main | ||
| paths-ignore: | ||
| - 'docs/**' | ||
| - 'example_config/**' | ||
| - 'docker/**' | ||
| - 'k8s_template/**' | ||
| - 'changes/**' | ||
| - 'licenses/**' | ||
| - 'CHANGELOG.md' | ||
| types: | ||
| - closed | ||
|
|
||
| jobs: | ||
| CI: | ||
| if: github.event.pull_request.merged == true | ||
| runs-on: ${{ matrix.runner }} | ||
| timeout-minutes: 60 | ||
| strategy: | ||
| matrix: | ||
| go-version: [ 1.19.10 ] | ||
| runner: [ ubuntu-latest ] | ||
| fail-fast: true | ||
| permissions: | ||
| contents: write | ||
| steps: | ||
| # Clean up space to prevent action from running out of disk space. | ||
| - name: clean | ||
| if: matrix.runner == 'ubuntu-latest' | ||
| run: | | ||
| sudo rm -rf /usr/share/dotnet | ||
| sudo rm -rf /opt/ghc | ||
| sudo rm -rf "/usr/local/share/boost" | ||
| sudo rm -rf "$AGENT_TOOLSDIRECTORY" | ||
| sudo -E apt-get -qq autoremove --purge | ||
| sudo -E apt-get -qq clean | ||
| - name: Check disk space | ||
| run: | | ||
| df -hT $PWD | ||
| - name: Set up Go ${{ matrix.go-version }} | ||
| uses: actions/setup-go@v2 | ||
| with: | ||
| go-version: ${{ matrix.go-version }} | ||
|
|
||
| - name: Check out code | ||
| uses: actions/checkout@v2 | ||
| with: | ||
| submodules: true | ||
|
|
||
| - name: Update Docker-compose to v2 | ||
| run: | | ||
| sudo curl -SL https://github.com/docker/compose/releases/download/v2.7.0/docker-compose-linux-x86_64 -o /usr/local/bin/docker-compose | ||
| sudo chmod +x /usr/local/bin/docker-compose | ||
| - name: System environment | ||
| run: | | ||
| uname -r | ||
| docker --version | ||
| go version | ||
| - name: Run benchmark | ||
| env: | ||
| BUILD_LOGTAIL_UT: OFF | ||
| WITHOUTGDB: ON | ||
| run: | | ||
| make benchmark | ||
| git stash | ||
| - name: Store benchmark result iLogtail | ||
| uses: benchmark-action/github-action-benchmark@v1 | ||
| with: | ||
| name: benchmark | ||
| tool: "customSmallerIsBetter" | ||
| output-file-path: "test/benchmark/report/ilogtail_statistic_all.json" | ||
| auto-push: false | ||
| summary-always: true | ||
|
|
||
| - name: Store benchmark result other agents | ||
| run: | | ||
| git checkout gh-pages | ||
| mv test/benchmark/report/records_all.json dev/bench/records_all.js | ||
| sed -i '1s/^\[/window.BENCHMARK_RECORDS = [/' dev/bench/records_all.js | ||
| git add dev/bench/records_all.js | ||
| last_commit_message=$(git log -1 --pretty=%B) | ||
| git -c user.name=github-action-benchmark -c [email protected] commit --amend -m "$last_commit_message" | ||
| # TODO: Push gh-pages branch by github-action-bot | ||
| # - name: Push benchmark result | ||
| # run: git push 'https://alibaba:${{ secrets.GITHUB_TOKEN }}@github.com/alibaba/ilogtail.git' gh-pages:gh-pages | ||
|
|
||
| result: | ||
| if: github.event.pull_request.merged == true | ||
| runs-on: ubuntu-latest | ||
| timeout-minutes: 60 | ||
| needs: [ CI ] | ||
| steps: | ||
| - name: Build Result | ||
| run: echo "Just to make the GitHub merge button green" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -28,6 +28,7 @@ on: | |
| branches: | ||
| - main | ||
| - 1.* | ||
| - benchmark | ||
| jobs: | ||
| CI: | ||
| env: | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,59 @@ | ||
| # Benchmark测试 | ||
|
|
||
| ## 工作原理 | ||
| 与e2e测试工作原理相同，详情可见： | ||
| - [e2e-test.md](./e2e-test.md) | ||
|
|
||
| ## 测试流程 | ||
|
|
||
| ### 环境准备 | ||
|
|
||
| 目前仅支持通过docker-compose搭建测试环境，在准备开始进行benchmark测试前，您首先需要准备以下内容： | ||
|
|
||
| - 测试环境：Docker-Compose环境（需在本地安装docker-compose） | ||
|
|
||
| ### 配置文件 | ||
|
|
||
| 对于每一个新的benchmark，您需要在`./test/benchmark/test_cases/<your_scenario>`目录下创建一个新的feature配置文件。每个配置文件中可以包含多个测试场景，每个测试场景由一个或多个步骤组成。 | ||
|
|
||
| 配置文件的基本框架如下： | ||
|
|
||
| ```plain | ||
| @input | ||
| Feature: performance file to blackhole vector | ||
| Performance file to blackhole vector | ||
| @e2e-performance @docker-compose | ||
| Scenario: PerformanceFileToBlackholeVector | ||
| Given {docker-compose} environment | ||
| Given docker-compose type {benchmark} | ||
| When start docker-compose {performance_file_to_blackhole_vector} | ||
| When start monitor {vector} | ||
| When generate random json logs to file, speed {10}MB/s, total {1}min, to file {./a.log} | ||
| When wait monitor until log processing finished | ||
| ``` | ||
|
|
||
| - `@e2e-performance @docker-compose`: 表示测试场景为e2e-performance，测试场景由本地docker-compose运行 | ||
| - `Given {docker-compose} environment`: 配置启动测试环境，以docker-compose环境启动测试 | ||
| - `Given docker-compose boot type {benchmark}`: 配置docker-compose启动模式，以benchmark模式启动docker-compose，`{}`中参数有两种选项，`e2e`/`benchmark`。以`e2e`模式启动会默认启动ilogtail、goc-server容器，用作e2e测试；以`benchmark`模式启动会默认启动cadvisor容器，用于监控容器运行过程中的资源占用；若在配置文件中不配置该参数，则默认以上一个scenario的启动模式启动。 | ||
| - `When start docker-compose {directory}`: `{}`中参数为当前scenario的文件夹名，该行动作会读取`directory`文件夹下的docker-compose.yaml文件，通过docker-compose命令启动所有容器 | ||
| - `When start monitor {vector}`: `{}`中参数为待监控的容器，该参数需要与docker-compose中的service name相同 | ||
| - `When generate random logs to file`: 向文件中按照速率生成json格式测试数据，其他生成测试数据的方法请参考[e2e-test-step.md](./e2e-test-step.md) | ||
| - `When wait monitor util log processing finished`: 等待采集agent日志收集完成 | ||
|
|
||
| ### 运行测试 | ||
|
|
||
| 在所有测试内容准备完毕后，您可以直接在test目录下以go test的方式运行E2E测试。根据您的需求，可以选择运行所有测试或者指定测试。 | ||
|
|
||
| ```shell | ||
| # 运行所有benchmark测试 | ||
| go test -v -timeout 30m -run ^TestE2EOnDockerComposePerformance$ github.com/alibaba/ilogtail/test/benchmark | ||
|
|
||
| # 以正则表达式匹配Scenario运行测试，以下为运行Scenario以Vector为结尾的test_case | ||
| go test -v -timeout 30m -run ^TestE2EOnDockerComposePerformance$/^.*Vector$ github.com/alibaba/ilogtail/test/benchmark | ||
| ``` | ||
|
|
||
| ### 测试结果 | ||
|
|
||
| - 所有统计结果将以json格式记录在`test/benchmark/report/<your_scenario>_statistic.json`中，目前记录了测试过程中CPU最大使用率、CPU平均使用率、内存最大使用率、内存平均使用率参数；所有实时结果序列将以json格式记录在`test/benchmark/report/<your_scenario>_records.json`中，目前记录了测试运行过程中的CPU使用率、内存使用率时间序列。 | ||
| - 运行`scripts/benchmark_collect_result.sh`会将数据以github benchmark action所需格式汇总，会将`test/benchmark/report/*ilogtail_statistic.json`下所有结果收集并生成汇总结果到`test/benchmark/report/ilogtail_statistic_all.json`中，并将`test/benchmark/report/*records.json`汇总到`test/benchmark/report/records_all.json` |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,49 @@ | ||
| #!/bin/bash | ||
|
|
||
| # Combine the statistic and records files from each benchmark run into a single file | ||
|
|
||
| # Define the input files and the output file | ||
| input_statistic_files=($(find test/benchmark/report -type f -name '*ilogtail_statistic.json')) | ||
| output_statistic_file="test/benchmark/report/ilogtail_statistic_all.json" | ||
|
|
||
| # Start the output file with an opening square bracket | ||
| rm -f "$output_statistic_file" | ||
| touch "$output_statistic_file" | ||
| echo '[' > "$output_statistic_file" | ||
|
|
||
| # Loop through each input file | ||
| for i in "${!input_statistic_files[@]}"; do | ||
| # Read the file, remove the first and last line (the square brackets), and append to the output file | ||
| cat "${input_statistic_files[$i]}" | sed '1d;$d' >> "$output_statistic_file" | ||
|
|
||
| # If this is not the last file, append a comma to separate the arrays | ||
| if [ $i -lt $((${#input_statistic_files[@]} - 1)) ]; then | ||
| echo ',' >> "$output_statistic_file" | ||
| fi | ||
| done | ||
|
|
||
| # Finish the output file with a closing square bracket | ||
| echo ']' >> "$output_statistic_file" | ||
|
|
||
| # Define the input files and the output file | ||
| input_records_files=($(find test/benchmark/report -type f -name '*records.json')) | ||
| output_records_file="test/benchmark/report/records_all.json" | ||
|
|
||
| # Start the output file with an opening square bracket | ||
| rm -f "$output_records_file" | ||
| touch "$output_records_file" | ||
| echo '[' > "$output_records_file" | ||
|
|
||
| # Loop through each input file | ||
| for i in "${!input_records_files[@]}"; do | ||
| # Read the file, remove the first and last line (the square brackets), and append to the output file | ||
| cat "${input_records_files[$i]}" | sed '1d;$d' >> "$output_records_file" | ||
|
|
||
| # If this is not the last file, append a comma to separate the arrays | ||
| if [ $i -lt $((${#input_records_files[@]} - 1)) ]; then | ||
| echo ',' >> "$output_records_file" | ||
| fi | ||
| done | ||
|
|
||
| # Finish the output file with a closing square bracket | ||
| echo ']' >> "$output_records_file" |
Oops, something went wrong.