Github runner add benchmark

.github/workflows/benchmark.yaml

@@ -0,0 +1,114 @@
+# Copyright 2021 iLogtail Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Benchmark
+
+on:
+  pull_request:
+    branches:
+      - main
+    paths-ignore:
+      - 'docs/**'
+      - 'example_config/**'
+      - 'docker/**'
+      - 'k8s_template/**'
+      - 'changes/**'
+      - 'licenses/**'
+      - 'CHANGELOG.md'
+    types:
+      - closed
+
+jobs:
+  CI:
+    if: github.event.pull_request.merged == true
+    runs-on: ${{ matrix.runner }}
+    timeout-minutes: 60
+    strategy:
+      matrix:
+        go-version: [ 1.19.10 ]
+        runner: [ ubuntu-latest ]
+      fail-fast: true
+    steps:
+      # Clean up space to prevent action from running out of disk space.
+      - name: clean
+        if: matrix.runner == 'ubuntu-latest'
+        run: |
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf /opt/ghc
+          sudo rm -rf "/usr/local/share/boost"
+          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+          sudo -E apt-get -qq autoremove --purge
+          sudo -E apt-get -qq clean
+
+      - name: Check disk space
+        run: |
+          df -hT $PWD
+
+      - name: Set up Go ${{ matrix.go-version }}
+        uses: actions/setup-go@v2
+        with:
+          go-version: ${{ matrix.go-version }}
+
+      - name: Check out code
+        uses: actions/checkout@v2
+        with:
+          submodules: true
+
+      - name: Update Docker-compose to v2
+        run: |
+          sudo curl -SL https://github.com/docker/compose/releases/download/v2.7.0/docker-compose-linux-x86_64 -o /usr/local/bin/docker-compose
+          sudo chmod +x /usr/local/bin/docker-compose
+
+      - name: System environment
+        run: |
+          uname -r
+          docker --version
+          go version
+
+      - name: Run benchmark
+        env:
+          BUILD_LOGTAIL_UT: OFF
+          WITHOUTGDB: ON
+        run: make benchmark
+
+      - name: Store benchmark result iLogtail
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          name: benchmark
+          tool: "customSmallerIsBetter"
+          output-file-path: "test/benchmark/report/ilogtail_statistic_all.json"
+          auto-push: false
+          summary-always: true
+
+      - name: Store benchmark result other agents
+        run: |
+          git checkout gh-pages
+          cp test/benchmark/report/records_all.json dev/bench/records_all.js
+          sed -i '1s/^\[/window.BENCHMARK_RECORDS = [/' dev/bench/records_all.js
+          git add dev/bench/records_all.js
+          last_commit_message=$(git log -1 --pretty=%B)
+          git -c user.name=github-action-benchmark -c [email protected] commit --amend -m "$last_commit_message"
+
+      # Push gh-pages branch by yourself
+      - name: Push benchmark result
+        run: git push 'https://alibaba:${{ secrets.GITHUB_TOKEN }}@github.com/alibaba/ilogtail.git' gh-pages:gh-pages
+
+  result:
+    if: github.event.pull_request.merged == true
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    needs: [ CI ]
+    steps:
+      - name: Build Result
+        run: echo "Just to make the GitHub merge button green"

.github/workflows/static-check.yaml

@@ -28,6 +28,7 @@ on:
     branches:
       - main
       - 1.*
+      - benchmark
 jobs:
   CI:
     env:

Makefile

@@ -181,12 +181,12 @@ e2e-core: clean gocdocker e2edocker
 	./scripts/e2e.sh e2e core
 
 .PHONY: e2e-performance
-e2e-performance: clean docker gocdocker
+e2e-performance: clean docker
 	./scripts/e2e.sh e2e performance
 
 .PHONY: unittest_e2e_engine
 unittest_e2e_engine: clean gocdocker
-	cd test && go test  $$(go list ./... | grep -Ev "engine|e2e") -coverprofile=../e2e-engine-coverage.txt -covermode=atomic -tags docker_ready
+	cd test && go test  $$(go list ./... | grep -Ev "engine|e2e|benchmark") -coverprofile=../e2e-engine-coverage.txt -covermode=atomic -tags docker_ready
 
 .PHONY: unittest_plugin
 unittest_plugin: clean import_plugins
@@ -210,6 +210,13 @@ unittest_pluginmanager: clean import_plugins
 	go test $$(go list ./...|grep -Ev "telegraf|external|envconfig"| grep -E "plugin_main|pluginmanager") -coverprofile .coretestCoverage.txt
 	mv ./plugins/input/prometheus/input_prometheus.go.bak ./plugins/input/prometheus/input_prometheus.go
 
+# benchmark
+.PHONY: benchmark
+benchmark: clean e2edocker
+	./scripts/e2e.sh benchmark performance
+	./scripts/benchmark_collect_result.sh
+
+
 .PHONY: all
 all: clean import_plugins
 	./scripts/gen_build_scripts.sh all "$(GENERATED_HOME)" "$(VERSION)" "$(BUILD_REPOSITORY)" "$(OUT_DIR)" "$(DOCKER_BUILD_EXPORT_GO_ENVS)" "$(DOCKER_BUILD_COPY_GIT_CONFIGS)" "$(PLUGINS_CONFIG_FILE)" "$(GO_MOD_FILE)"

docs/cn/developer-guide/test/benchmark.md

@@ -0,0 +1,60 @@
+# Benchmark测试
+
+## 工作原理
+与e2e测试工作原理相同，详情可见：
+- [e2e-test.md](./e2e-test.md)
+
+## 测试流程
+
+### 环境准备
+
+目前仅支持通过docker-compose搭建测试环境，在准备开始进行benchmark测试前，您首先需要准备以下内容：
+
+- 测试环境：Docker-Compose环境（需在本地安装docker-compose）
+
+### 配置文件
+
+对于每一个新的benchmark，您需要在`./test/benchmark/test_cases/<your_scenario>`目录下创建一个新的feature配置文件。每个配置文件中可以包含多个测试场景，每个测试场景由一个或多个步骤组成。
+
+配置文件的基本框架如下：
+
+```plain
+@input
+Feature: performance file to blackhole vector
+  Performance file to blackhole vector
+
+  @e2e-performance @docker-compose
+  Scenario: PerformanceFileToBlackholeVector
+    Given {docker-compose} environment
+    Given docker-compose type {benchmark}
+    When start docker-compose {performance_file_to_blackhole_vector}
+    When start monitor {vector}
+    When generate logs to file, speed {10}MB/s, total {1}min, to file {./a.log}, template
+    """
+    {"url": "POST /PutData?Category=YunOsAccountOpLog HTTP/1.1", "ip": "10.200.98.220", "user-agent": "aliyun-sdk-java", "request": {"status": "200", "latency": "18204"}, "time": "07/Jul/2022:10:30:28"}
+    """
+```
+
+- `@e2e-performance @docker-compose`: 表示测试场景为e2e-performance，测试场景由本地docker-compose运行
+- `Given {docker-compose} environment`: 配置启动测试环境，以docker-compose环境启动测试
+- `Given docker-compose boot type {benchmark}`: 配置docker-compose启动模式，以benchmark模式启动docker-compose，`{}`中参数有两种选项，`e2e`/`benchmark`。以`e2e`模式启动会默认启动ilogtail、goc-server容器，用作e2e测试；以`benchmark`模式启动会默认启动cadvisor容器，用于监控容器运行过程中的资源占用；若在配置文件中不配置该参数，则默认以上一个scenario的启动模式启动。
+- `When start docker-compose {directory}`: `{}`中参数为当前scenario的文件夹名，该行动作会读取`directory`文件夹下的docker-compose.yaml文件，通过docker-compose命令启动所有容器
+- `When start monitor {vector}`: `{}`中参数为待监控的容器，该参数需要与docker-compose中的service name相同
+- `When generate logs to file`: 生成测试数据，其他生成测试数据的方法请参考[e2e-test-step.md](./e2e-test-step.md)
+
+### 运行测试
+
+在所有测试内容准备完毕后，您可以直接在test目录下以go test的方式运行E2E测试。根据您的需求，可以选择运行所有测试或者指定测试。
+
+```shell
+# 运行所有benchmark测试
+go test -v -timeout 30m -run ^TestE2EOnDockerComposePerformance$ github.com/alibaba/ilogtail/test/benchmark
+
+# 以正则表达式匹配Scenario运行测试，以下为运行Scenario以Vector为结尾的test_case
+go test -v -timeout 30m -run ^TestE2EOnDockerComposePerformance$/^.*Vector$ github.com/alibaba/ilogtail/test/benchmark
+```
+
+### 测试结果
+
+- 所有统计结果将以json格式记录在`test/benchmark/report/<your_scenario>_statistic.json`中，目前记录了测试过程中CPU最大使用率、CPU平均使用率、内存最大使用率、内存平均使用率参数；所有实时结果序列将以json格式记录在`test/benchmark/report/<your_scenario>_records.json`中，目前记录了测试运行过程中的CPU使用率、内存使用率时间序列。
+- 运行`scripts/benchmark_collect_result.sh`会将数据以github benchmark action所需格式汇总，会将`test/benchmark/report/*ilogtail_statistic.json`下所有结果收集并生成汇总结果到`test/benchmark/report/ilogtail_statistic_all.json`中，并将`test/benchmark/report/*records.json`汇总到`test/benchmark/report/records_all.json`

licenses/LICENSE_OF_TESTENGINE_DEPENDENCIES.md

@@ -16,7 +16,6 @@ When distributed in a binary form, Logtailplugin Test Engine may contain portion
 - [github.com/opencontainers/go-digest](https://pkg.go.dev/github.com/opencontainers/go-digest?tab=licenses)
 - [github.com/opencontainers/image-spec](https://pkg.go.dev/github.com/opencontainers/image-spec?tab=licenses)
 - [github.com/opencontainers/runc](https://pkg.go.dev/github.com/opencontainers/runc?tab=licenses)
-- [google.golang.org/genproto](https://pkg.go.dev/google.golang.org/genproto?tab=licenses)
 - [google.golang.org/grpc](https://pkg.go.dev/google.golang.org/grpc?tab=licenses)
 - [gopkg.in/ini.v1](https://pkg.go.dev/gopkg.in/ini.v1?tab=licenses)
 - [gopkg.in/yaml.v2](https://pkg.go.dev/gopkg.in/yaml.v2?tab=licenses)
@@ -45,10 +44,12 @@ When distributed in a binary form, Logtailplugin Test Engine may contain portion
 - [sigs.k8s.io/json](https://pkg.go.dev/sigs.k8s.io/json?tab=licenses)
 - [sigs.k8s.io/structured-merge-diff](https://pkg.go.dev/sigs.k8s.io/structured-merge-diff?tab=licenses)
 - [sigs.k8s.io/yaml](https://pkg.go.dev/sigs.k8s.io/yaml?tab=licenses)
+- [github.com/google/cadvisor](https://pkg.go.dev/github.com/google/cadvisor?tab=licenses)
 - [github.com/go-openapi/jsonpointer](https://pkg.go.dev/github.com/go-openapi/jsonpointer?tab=licenses)
 - [github.com/go-openapi/jsonreference](https://pkg.go.dev/github.com/go-openapi/jsonreference?tab=licenses)
 - [github.com/go-openapi/swag](github.com/go-openapi/swag)
 - [github.com/google/gnostic](https://pkg.go.dev/github.com/google/gnostic?tab=licenses)
+- [google.golang.org/genproto/googleapis/rpc](https://pkg.go.dev/google.golang.org/genproto/googleapis/rpc?tab=licenses)
 
 ## BSD licenses
 

scripts/benchmark_collect_result.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+
+# Combine the statistic and records files from each benchmark run into a single file
+
+# Define the input files and the output file
+input_statistic_files=($(find test/benchmark/report -type f -name '*ilogtail_statistic.json'))
+output_statistic_file="test/benchmark/report/ilogtail_statistic_all.json"
+
+# Start the output file with an opening square bracket
+rm -f "$output_statistic_file"
+touch "$output_statistic_file"
+echo '[' > "$output_statistic_file"
+
+# Loop through each input file
+for i in "${!input_statistic_files[@]}"; do
+  # Read the file, remove the first and last line (the square brackets), and append to the output file
+  cat "${input_statistic_files[$i]}" | sed '1d;$d' >> "$output_statistic_file"
+
+  # If this is not the last file, append a comma to separate the arrays
+  if [ $i -lt $((${#input_statistic_files[@]} - 1)) ]; then
+    echo ',' >> "$output_statistic_file"
+  fi
+done
+
+# Finish the output file with a closing square bracket
+echo ']' >> "$output_statistic_file"
+
+# Define the input files and the output file
+input_records_files=($(find test/benchmark/report -type f -name '*records.json'))
+output_records_file="test/benchmark/report/records_all.json"
+
+# Start the output file with an opening square bracket
+rm -f "$output_records_file"
+touch "$output_records_file"
+echo '[' > "$output_records_file"
+
+# Loop through each input file
+for i in "${!input_records_files[@]}"; do
+  # Read the file, remove the first and last line (the square brackets), and append to the output file
+  cat "${input_records_files[$i]}" | sed '1d;$d' >> "$output_records_file"
+
+  # If this is not the last file, append a comma to separate the arrays
+  if [ $i -lt $((${#input_records_files[@]} - 1)) ]; then
+    echo ',' >> "$output_records_file"
+  fi
+done
+
+# Finish the output file with a closing square bracket
+echo ']' >> "$output_records_file"

test/benchmark/benchmark_docker_test.go

@@ -0,0 +1,38 @@
+// Copyright 2024 iLogtail Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package e2e
+
+import (
+	"testing"
+
+	"github.com/cucumber/godog"
+
+	"github.com/alibaba/ilogtail/test/engine"
+)
+
+func TestE2EOnDockerComposePerformance(t *testing.T) {
+	suite := godog.TestSuite{
+		Name:                "E2EOnDockerCompose",
+		ScenarioInitializer: engine.ScenarioInitializer,
+		Options: &godog.Options{
+			Format:   "pretty",
+			Paths:    []string{"test_cases"},
+			Tags:     "@e2e-performance && @docker-compose && ~@ebpf",
+			TestingT: t,
+		},
+	}
+	if suite.Run() != 0 {
+		t.Fail()
+	}
+}

test/benchmark/test_cases/performance_file_to_blackhole_filebeat/case.feature

@@ -0,0 +1,14 @@
+@input
+Feature: performance file to blackhole filebeat
+  Performance file to blackhole filebeat
+
+  @e2e-performance @docker-compose
+  Scenario: PerformanceFileToBlackholeFilebeat
+    Given {docker-compose} environment
+    Given docker-compose boot type {benchmark}
+    When start docker-compose {performance_file_to_blackhole_filebeat}
+    When start monitor {filebeat}
+    When generate logs to file, speed {100}MB/s, total {1}min, to file {./a.log}, template
+    """
+    {"url": "POST /PutData?Category=YunOsAccountOpLog HTTP/1.1", "ip": "10.200.98.220", "user-agent": "aliyun-sdk-java", "request": {"status": "200", "latency": "18204"}, "time": "07/Jul/2022:10:30:28"}
+    """

test/benchmark/test_cases/performance_file_to_blackhole_filebeat/docker-compose.yaml

@@ -0,0 +1,10 @@
+version: '3.8'
+
+services:
+  filebeat:
+    image: docker.elastic.co/beats/filebeat:8.14.2
+    user: root
+    volumes:
+      - ./filebeat.yml:/usr/share/filebeat/filebeat.yml:ro
+      - .:/home/filebeat
+    command: filebeat -e --strict.perms=false

test/benchmark/test_cases/performance_file_to_blackhole_filebeat/filebeat.yml

@@ -0,0 +1,19 @@
+filebeat.inputs:
+- type: filestream
+  id: input-file
+  paths:
+    - /home/filebeat/*.log
+  prospector.scanner.check_interval: 1s
+
+processors:
+  - decode_json_fields:
+      fields: ["message"]
+      target: "json"
+  - drop_event:
+      when:
+        not:
+          equals:
+            json.user-agent: "no-agent"
+
+output.console:
+  pretty: true

test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/case.feature

@@ -0,0 +1,14 @@
+@input
+Feature: performance file to blackhole fluentbit
+  Performance file to blackhole fluentbit
+
+  @e2e-performance @docker-compose
+  Scenario: PerformanceFileToBlackholeFluentbit
+    Given {docker-compose} environment
+    Given docker-compose boot type {benchmark}
+    When start docker-compose {performance_file_to_blackhole_fluentbit}
+    When start monitor {fluent-bit}
+    When generate logs to file, speed {100}MB/s, total {1}min, to file {./a.log}, template
+    """
+    {"url": "POST /PutData?Category=YunOsAccountOpLog HTTP/1.1", "ip": "10.200.98.220", "user-agent": "aliyun-sdk-java", "request": {"status": "200", "latency": "18204"}, "time": "07/Jul/2022:10:30:28"}
+    """

test/benchmark/test_cases/performance_file_to_blackhole_fluentbit/docker-compose.yaml

@@ -0,0 +1,14 @@
+version: '3.8'
+
+services:
+
+  fluent-bit:
+    image: cr.fluentbit.io/fluent/fluent-bit
+    command: ["-c", "/tmp/main.conf"]
+    volumes:
+      - ./main.conf:/tmp/main.conf
+      - ./parsers.conf:/tmp/parsers.conf
+      - .:/home/fluentbit
+    restart: always
+
+