From bf804788796b3c9b319d51ae4b8da0b1afd9024e Mon Sep 17 00:00:00 2001 From: spacewander Date: Wed, 23 Oct 2024 09:51:15 +0800 Subject: [PATCH] test: avoid being conflict with well-known ports Signed-off-by: spacewander --- .github/workflows/test.yml | 5 +++ .../integration/controlplane/control_plane.go | 13 ++++++- .../tests/integration/dataplane/bootstrap.go | 14 ++++++++ .../tests/integration/dataplane/data_plane.go | 36 ++++++++++++++----- .../integration/filtermanager_latest_test.go | 2 +- .../integration/testdata/grpc_backend.yml | 2 +- .../testdata/services/docker-compose.yml | 2 +- .../testdata/services/grpc/main.go | 4 +-- .../plugin_integration_test_framework.md | 11 +++--- .../plugin_integration_test_framework.md | 11 +++--- 10 files changed, 75 insertions(+), 25 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9b459c98..7631bae1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -153,6 +153,11 @@ jobs: plugins-integration-test: timeout-minutes: 10 runs-on: ubuntu-latest + env: + # to test the custom port feature + TEST_ENVOY_ADMIN_API_PORT: 9901 + TEST_ENVOY_CONTROL_PLANE_PORT: 9902 + TEST_ENVOY_DATA_PLANE_PORT: 9903 defaults: run: working-directory: ./plugins diff --git a/api/plugins/tests/integration/controlplane/control_plane.go b/api/plugins/tests/integration/controlplane/control_plane.go index 92dcf7a2..41967cb1 100644 --- a/api/plugins/tests/integration/controlplane/control_plane.go +++ b/api/plugins/tests/integration/controlplane/control_plane.go @@ -99,7 +99,18 @@ func (cp *ControlPlane) Start() { // only use it on Linux. } - lis, _ := net.Listen("tcp", host+":9999") + port := ":9999" + portEnv := os.Getenv("TEST_ENVOY_CONTROL_PLANE_PORT") + if portEnv != "" { + port = ":" + portEnv + } + + lis, err := net.Listen("tcp", host+port) + if err != nil { + logger.Error(err, "failed to listen") + return + } + if err := cp.grpcServer.Serve(lis); err != nil { logger.Error(err, "failed to start control plane") } diff --git a/api/plugins/tests/integration/dataplane/bootstrap.go b/api/plugins/tests/integration/dataplane/bootstrap.go index 105d48e2..21ffad7e 100644 --- a/api/plugins/tests/integration/dataplane/bootstrap.go +++ b/api/plugins/tests/integration/dataplane/bootstrap.go @@ -149,6 +149,20 @@ func (b *bootstrap) buildConfiguration() (map[string]interface{}, error) { staticResources := root["static_resources"].(map[string]interface{}) clusters := staticResources["clusters"].([]interface{}) + + port := "9999" + portEnv := os.Getenv("TEST_ENVOY_CONTROL_PLANE_PORT") + if portEnv != "" { + port = portEnv + } + for _, c := range clusters { + if c.(map[string]interface{})["name"] == "config_server" { + load := c.(map[string]interface{})["load_assignment"].(map[string]interface{})["endpoints"].([]interface{})[0].(map[string]interface{})["lb_endpoints"].([]interface{})[0].(map[string]interface{})["endpoint"].(map[string]interface{})["address"].(map[string]interface{})["socket_address"].(map[string]interface{}) + load["port_value"] = port + break + } + } + newClusters := []interface{}{} for _, c := range b.clusters { newClusters = append(newClusters, c) diff --git a/api/plugins/tests/integration/dataplane/data_plane.go b/api/plugins/tests/integration/dataplane/data_plane.go index 5d3a79b3..44af8d0c 100644 --- a/api/plugins/tests/integration/dataplane/data_plane.go +++ b/api/plugins/tests/integration/dataplane/data_plane.go @@ -54,6 +54,9 @@ type DataPlane struct { t *testing.T opt *Option done chan error + + dataPlanePort string + adminAPIPort string } type Option struct { @@ -189,6 +192,20 @@ func StartDataPlane(t *testing.T, opt *Option) (*DataPlane, error) { return nil, err } + adminAPIPort := "9998" + adminAPIPortEnv := os.Getenv("TEST_ENVOY_ADMIN_API_PORT") + if adminAPIPortEnv != "" { + adminAPIPort = adminAPIPortEnv + } + dp.adminAPIPort = adminAPIPort + + dataPlanePort := "10000" + dataPlanePortEnv := os.Getenv("TEST_ENVOY_DATA_PLANE_PORT") + if dataPlanePortEnv != "" { + dataPlanePort = dataPlanePortEnv + } + dp.dataPlanePort = dataPlanePort + cmdline := "docker run" + " --name " + containerName + " --network " + networkName + @@ -199,7 +216,8 @@ func StartDataPlane(t *testing.T, opt *Option) (*DataPlane, error) { " -v /tmp:/tmp" + " -e GOCOVERDIR=" + coverDir + " " + strings.Join(envs, " ") + - " -p 10000:10000 -p 9998:9998 " + hostAddr + " " + + " -p " + dataPlanePort + ":10000 -p " + adminAPIPort + ":9998 " + + hostAddr + " " + image content, _ := os.ReadFile(cfgFile.Name()) @@ -256,7 +274,7 @@ func StartDataPlane(t *testing.T, opt *Option) (*DataPlane, error) { go func() { done <- cmd.Wait() }() }() - helper.WaitServiceUp(t, ":10000", "") + helper.WaitServiceUp(t, ":"+dataPlanePort, "") select { case err := <-done: @@ -381,7 +399,7 @@ func (dp *DataPlane) Patch(path string, header http.Header, body io.Reader) (*ht } func (dp *DataPlane) SendAndCancelRequest(path string, after time.Duration) error { - conn, err := net.DialTimeout("tcp", ":10000", 1*time.Second) + conn, err := net.DialTimeout("tcp", ":"+dp.dataPlanePort, 1*time.Second) if err != nil { return err } @@ -404,13 +422,13 @@ func (dp *DataPlane) SendAndCancelRequest(path string, after time.Duration) erro } func (dp *DataPlane) do(method string, path string, header http.Header, body io.Reader) (*http.Response, error) { - req, err := http.NewRequest(method, "http://localhost:10000"+path, body) + req, err := http.NewRequest(method, "http://localhost:"+dp.dataPlanePort+path, body) if err != nil { return nil, err } req.Header = header tr := &http.Transport{DialContext: func(ctx context.Context, proto, addr string) (conn net.Conn, err error) { - return net.DialTimeout("tcp", ":10000", 1*time.Second) + return net.DialTimeout("tcp", ":"+dp.dataPlanePort, 1*time.Second) }} client := &http.Client{Transport: tr, @@ -424,7 +442,7 @@ func (dp *DataPlane) do(method string, path string, header http.Header, body io. } func (dp *DataPlane) doWithTrailer(method string, path string, header http.Header, body io.Reader, trailer http.Header) (*http.Response, error) { - req, err := http.NewRequest(method, "http://localhost:10000"+path, body) + req, err := http.NewRequest(method, "http://localhost:"+dp.dataPlanePort+path, body) if err != nil { return nil, err } @@ -434,7 +452,7 @@ func (dp *DataPlane) doWithTrailer(method string, path string, header http.Heade req.TransferEncoding = []string{"chunked"} tr := &http.Transport{ DialContext: func(ctx context.Context, proto, addr string) (conn net.Conn, err error) { - return net.DialTimeout("tcp", ":10000", 1*time.Second) + return net.DialTimeout("tcp", ":"+dp.dataPlanePort, 1*time.Second) }, } @@ -451,7 +469,7 @@ func (dp *DataPlane) doWithTrailer(method string, path string, header http.Heade // Use grpcurl so that the caller can specify the proto file without building the Go code. // TODO: we can rewrite this in Go. func (dp *DataPlane) Grpcurl(importPath, protoFile, fullMethodName, req string) ([]byte, error) { - cmd := exec.Command("grpcurl", "-v", "-format-error", "-import-path", importPath, "-proto", protoFile, "-plaintext", "-d", req, ":10000", fullMethodName) + cmd := exec.Command("grpcurl", "-v", "-format-error", "-import-path", importPath, "-proto", protoFile, "-plaintext", "-d", req, ":"+dp.dataPlanePort, fullMethodName) dp.t.Logf("run grpcurl command: %s", cmd.String()) return cmd.CombinedOutput() } @@ -479,7 +497,7 @@ func (dp *DataPlane) FlushCoverage() error { } func (dp *DataPlane) SetLogLevel(loggerName string, level string) error { - req, err := http.NewRequest("POST", fmt.Sprintf("http://0.0.0.0:9998/logging?%s=%s", loggerName, level), bytes.NewReader([]byte{})) + req, err := http.NewRequest("POST", fmt.Sprintf("http://0.0.0.0:%s/logging?%s=%s", dp.adminAPIPort, loggerName, level), bytes.NewReader([]byte{})) if err != nil { return err } diff --git a/api/tests/integration/filtermanager_latest_test.go b/api/tests/integration/filtermanager_latest_test.go index 6af1a0bc..4e56616f 100644 --- a/api/tests/integration/filtermanager_latest_test.go +++ b/api/tests/integration/filtermanager_latest_test.go @@ -242,7 +242,7 @@ func TestFilterManagerTrailersWithGrpcBackend(t *testing.T) { } defer dp.Stop() - helper.WaitServiceUp(t, ":50051", "grpc") + helper.WaitServiceUp(t, ":50001", "grpc") s := &filtermanager.FilterManagerConfig{ Plugins: []*model.FilterConfig{ diff --git a/api/tests/integration/testdata/grpc_backend.yml b/api/tests/integration/testdata/grpc_backend.yml index 4b35d155..a11e33ed 100644 --- a/api/tests/integration/testdata/grpc_backend.yml +++ b/api/tests/integration/testdata/grpc_backend.yml @@ -10,4 +10,4 @@ load_assignment: address: socket_address: address: grpc - port_value: 50051 + port_value: 50001 diff --git a/api/tests/integration/testdata/services/docker-compose.yml b/api/tests/integration/testdata/services/docker-compose.yml index 534b2201..94704bf9 100644 --- a/api/tests/integration/testdata/services/docker-compose.yml +++ b/api/tests/integration/testdata/services/docker-compose.yml @@ -3,7 +3,7 @@ services: grpc: build: ./grpc ports: - - "50051:50051" + - "50001:50001" restart: unless-stopped networks: service: diff --git a/api/tests/integration/testdata/services/grpc/main.go b/api/tests/integration/testdata/services/grpc/main.go index 86084d3f..c68f2513 100644 --- a/api/tests/integration/testdata/services/grpc/main.go +++ b/api/tests/integration/testdata/services/grpc/main.go @@ -30,7 +30,7 @@ func (s *sampleServer) Ouch(ctx context.Context, req *HelloRequest) (*HelloRespo func main() { // Create a TCP listener - lis, err := net.Listen("tcp", ":50051") + lis, err := net.Listen("tcp", ":50001") if err != nil { log.Fatalf("Failed to listen: %v", err) } @@ -42,7 +42,7 @@ func main() { RegisterSampleServer(s, &sampleServer{}) // Start the server - fmt.Println("Server started. Listening on port :50051") + fmt.Println("Server started. Listening on port :50001") if err := s.Serve(lis); err != nil { log.Fatalf("Failed to serve: %v", err) } diff --git a/site/content/en/docs/developer-guide/plugin_integration_test_framework.md b/site/content/en/docs/developer-guide/plugin_integration_test_framework.md index 8fbce53c..f11f180b 100644 --- a/site/content/en/docs/developer-guide/plugin_integration_test_framework.md +++ b/site/content/en/docs/developer-guide/plugin_integration_test_framework.md @@ -20,9 +20,10 @@ You may have noticed that when executing `go test`, we added `-tags envoy1.29`. ## Port usage -The test framework will use: +The test framework will occupy the following ports on the host machine: -* `:2023` to represent invalid port -* `:9999` for the control plane -* `:10000` for the Envoy proxy -* `:10001` for the backend server and mock external server +* `:9998` for the Envoy's Admin API, which can be modified by the environment variable `TEST_ENVOY_ADMIN_API_PORT` +* `:9999` for the control plane, which can be modified by the environment variable `TEST_ENVOY_CONTROL_PLANE_PORT` +* `:10000` for the Envoy proxy, which can be modified by the environment variable `TEST_ENVOY_DATA_PLANE_PORT` + +For example, `TEST_ENVOY_CONTROL_PLANE_PORT=19999 go test -v ./tests/integration -run TestPluginXX` will use `:19999` as the control plane port. diff --git a/site/content/zh-hans/docs/developer-guide/plugin_integration_test_framework.md b/site/content/zh-hans/docs/developer-guide/plugin_integration_test_framework.md index f67b1b6d..e4d5c26a 100644 --- a/site/content/zh-hans/docs/developer-guide/plugin_integration_test_framework.md +++ b/site/content/zh-hans/docs/developer-guide/plugin_integration_test_framework.md @@ -20,9 +20,10 @@ title: 插件集成测试框架 ## 端口使用 -测试框架将使用： +测试框架将占用 host 上的下述端口： -* `:2023` 用于表示错误的端口 -* `:9999` 用于控制平面 -* `:10000` 用于数据面 -* `:10001` 用于后端服务器和模拟外部服务器 +* `:9998` 用于 Envoy 管理 API，可通过环境变量 `TEST_ENVOY_ADMIN_API_PORT` 修改 +* `:9999` 用于控制平面，可通过环境变量 `TEST_ENVOY_CONTROL_PLANE_PORT` 修改 +* `:10000` 用于数据面，可通过环境变量 `TEST_ENVOY_DATA_PLANE_PORT` 修改 + +例如，`TEST_ENVOY_CONTROL_PLANE_PORT=19999 go test -v ./tests/integration -run TestPluginXX` 将使用 `:19999` 端口作为控制平面端口。