From 935228465dd3c44ab02fead6750ba9bf11d30f6a Mon Sep 17 00:00:00 2001 From: Romain Dauby Date: Fri, 20 Dec 2024 16:07:15 -0500 Subject: [PATCH 01/13] Set GOMAXPROCS in AWS ECS task Signed-off-by: Romain Dauby --- extension/cgroupruntimeextension/README.md | 2 +- extension/cgroupruntimeextension/factory.go | 15 ++++++++++++--- extension/cgroupruntimeextension/go.mod | 5 ++++- extension/cgroupruntimeextension/go.sum | 2 ++ 4 files changed, 19 insertions(+), 5 deletions(-) diff --git a/extension/cgroupruntimeextension/README.md b/extension/cgroupruntimeextension/README.md index f7d79099d6c1..995be9cf314c 100644 --- a/extension/cgroupruntimeextension/README.md +++ b/extension/cgroupruntimeextension/README.md @@ -15,7 +15,7 @@ ## Overview -The OpenTelemetry Cgroup Auto-Config Extension is designed to optimize Go runtime performance in containerized environments by automatically configuring GOMAXPROCS and GOMEMLIMIT based on the Linux cgroup filesystem. This extension leverages [automaxprocs](https://github.com/uber-go/automaxprocs) and [automemlimit](https://github.com/KimMachineGun/automemlimit) packages to dynamically adjust Go runtime variables, ensuring efficient resource usage aligned with container limits. +The OpenTelemetry Cgroup Auto-Config Extension is designed to optimize Go runtime performance in containerized environments by automatically configuring GOMAXPROCS and GOMEMLIMIT based on the Linux cgroup filesystem. This extension leverages [automaxprocs](https://github.com/uber-go/automaxprocs) or [gomaxecs](https://github.com/rdforte/gomaxecs) for AWS ECS Tasks and [automemlimit](https://github.com/KimMachineGun/automemlimit) packages to dynamically adjust Go runtime variables, ensuring efficient resource usage aligned with container limits. ## Configuration diff --git a/extension/cgroupruntimeextension/factory.go b/extension/cgroupruntimeextension/factory.go index 1905811dc3b8..4079c89c4021 100644 --- a/extension/cgroupruntimeextension/factory.go +++ b/extension/cgroupruntimeextension/factory.go @@ -9,6 +9,7 @@ import ( "runtime/debug" "github.com/KimMachineGun/automemlimit/memlimit" + gomaxecs "github.com/rdforte/gomaxecs/maxprocs" "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/extension" "go.uber.org/automaxprocs/maxprocs" @@ -42,9 +43,17 @@ func createExtension(_ context.Context, set extension.Settings, cfg component.Co cgroupConfig := cfg.(*Config) return newCgroupRuntime(cgroupConfig, set.Logger, func() (undoFunc, error) { - undo, err := maxprocs.Set(maxprocs.Logger(func(str string, params ...any) { - set.Logger.Debug(fmt.Sprintf(str, params)) - })) + var undo func() + var err error + if gomaxecs.IsECS() { + undo, err = gomaxecs.Set(gomaxecs.WithLogger(func(str string, params ...any) { + set.Logger.Debug(fmt.Sprintf(str, params)) + })) + } else { + undo, err = maxprocs.Set(maxprocs.Logger(func(str string, params ...any) { + set.Logger.Debug(fmt.Sprintf(str, params)) + })) + } return undoFunc(undo), err }, func(ratio float64) (undoFunc, error) { diff --git a/extension/cgroupruntimeextension/go.mod b/extension/cgroupruntimeextension/go.mod index 8b35bfe962ff..46784ad9ebbf 100644 --- a/extension/cgroupruntimeextension/go.mod +++ b/extension/cgroupruntimeextension/go.mod @@ -1,10 +1,13 @@ module github.com/open-telemetry/opentelemetry-collector-contrib/extension/cgroupruntimeextension -go 1.22.0 +go 1.22.4 + +toolchain go1.22.5 require ( github.com/KimMachineGun/automemlimit v0.7.0 github.com/containerd/cgroups/v3 v3.0.5 + github.com/rdforte/gomaxecs v1.1.0 github.com/stretchr/testify v1.10.0 go.opentelemetry.io/collector/component v0.116.1-0.20241220212031-7c2639723f67 go.opentelemetry.io/collector/component/componenttest v0.116.1-0.20241220212031-7c2639723f67 diff --git a/extension/cgroupruntimeextension/go.sum b/extension/cgroupruntimeextension/go.sum index 0648233b2d7e..7a7fe976b339 100644 --- a/extension/cgroupruntimeextension/go.sum +++ b/extension/cgroupruntimeextension/go.sum @@ -63,6 +63,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g= github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U= +github.com/rdforte/gomaxecs v1.1.0 h1:fpDkJtuBRtRQjcxAKdARjwjYzxlmmGkSmcqzF0UKuOg= +github.com/rdforte/gomaxecs v1.1.0/go.mod h1:8agrawOmcvb+oBa6EnV2oADDtnDtkVx1Q0H/Ht7GiFc= github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= From c212117890984c304bf234bf20c11897aed7a54d Mon Sep 17 00:00:00 2001 From: Romain Dauby Date: Fri, 20 Dec 2024 16:20:36 -0500 Subject: [PATCH 02/13] Try integration test Signed-off-by: Romain Dauby --- .../integration_test.go | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/extension/cgroupruntimeextension/integration_test.go b/extension/cgroupruntimeextension/integration_test.go index 1dd8c89647ff..56f356e89fd7 100644 --- a/extension/cgroupruntimeextension/integration_test.go +++ b/extension/cgroupruntimeextension/integration_test.go @@ -9,8 +9,11 @@ package cgroupruntimeextension // import "github.com/open-telemetry/opentelemetr import ( "context" + "encoding/json" "fmt" "math" + "net/http" + "net/http/httptest" "os" "path" "path/filepath" @@ -63,6 +66,21 @@ func cgroupMaxCPU(filename string) (quota int64, period uint64, err error) { return quota, period, err } +func startMockECSServer() *httptest.Server { + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + response := map[string]interface{}{ + "DockerID": "container-id", + "Limits": map[string]interface{}{ + "CPU": 2.0, + }, + } + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(response) + }) + + return httptest.NewServer(handler) +} + func TestCgroupV2SudoIntegration(t *testing.T) { checkCgroupSystem(t) pointerInt64 := func(val int64) *int64 { @@ -81,6 +99,7 @@ func TestCgroupV2SudoIntegration(t *testing.T) { config *Config expectedGoMaxProcs int expectedGoMemLimit int64 + setECSMetadataURI bool }{ { name: "90% the max cgroup memory and 12 GOMAXPROCS", @@ -144,6 +163,24 @@ func TestCgroupV2SudoIntegration(t *testing.T) { // 134217728 * 0.1 expectedGoMemLimit: 13421772, }, + { + name: "running on AWS ECS with 90% of max cgroup memory and 2 GOMAXPROCS", + cgroupCpuQuota: pointerInt64(-1), + cgroupCpuPeriod: 8000, + cgroupMaxMemory: 134217728, // 128 MB + config: &Config{ + GoMaxProcs: GoMaxProcsConfig{ + Enabled: true, + }, + GoMemLimit: GoMemLimitConfig{ + Enabled: true, + Ratio: 0.9, + }, + }, + expectedGoMaxProcs: 22, + expectedGoMemLimit: 120795955, // 134217728 * 0.9 + setECSMetadataURI: true, + }, } cgroupPath, err := cgroup2.PidGroupPath(os.Getpid()) @@ -220,6 +257,13 @@ func TestCgroupV2SudoIntegration(t *testing.T) { }) require.NoError(t, err) + if test.setECSMetadataURI { + server := startMockECSServer() + defer server.Close() + os.Setenv("ECS_CONTAINER_METADATA_URI_V4", server.URL) + defer os.Unsetenv("ECS_CONTAINER_METADATA_URI_V4") + } + factory := NewFactory() ctx := context.Background() extension, err := factory.Create(ctx, extensiontest.NewNopSettings(), test.config) From cc6a96c448d51b36e2dc54c5cd52f6f8bf9d263a Mon Sep 17 00:00:00 2001 From: Romain Dauby Date: Fri, 20 Dec 2024 16:42:06 -0500 Subject: [PATCH 03/13] Update mod to go v1.22.4 --- extension/cgroupruntimeextension/go.mod | 2 -- 1 file changed, 2 deletions(-) diff --git a/extension/cgroupruntimeextension/go.mod b/extension/cgroupruntimeextension/go.mod index 46784ad9ebbf..6988015f04b4 100644 --- a/extension/cgroupruntimeextension/go.mod +++ b/extension/cgroupruntimeextension/go.mod @@ -2,8 +2,6 @@ module github.com/open-telemetry/opentelemetry-collector-contrib/extension/cgrou go 1.22.4 -toolchain go1.22.5 - require ( github.com/KimMachineGun/automemlimit v0.7.0 github.com/containerd/cgroups/v3 v3.0.5 From 443b93c24b7fd33e11d76ced983ee57589e11290 Mon Sep 17 00:00:00 2001 From: Romain Dauby Date: Thu, 26 Dec 2024 17:31:09 -0500 Subject: [PATCH 04/13] Refacto return logic --- extension/cgroupruntimeextension/factory.go | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/extension/cgroupruntimeextension/factory.go b/extension/cgroupruntimeextension/factory.go index 4079c89c4021..fdc6fca6fe12 100644 --- a/extension/cgroupruntimeextension/factory.go +++ b/extension/cgroupruntimeextension/factory.go @@ -43,18 +43,14 @@ func createExtension(_ context.Context, set extension.Settings, cfg component.Co cgroupConfig := cfg.(*Config) return newCgroupRuntime(cgroupConfig, set.Logger, func() (undoFunc, error) { - var undo func() - var err error if gomaxecs.IsECS() { - undo, err = gomaxecs.Set(gomaxecs.WithLogger(func(str string, params ...any) { - set.Logger.Debug(fmt.Sprintf(str, params)) - })) - } else { - undo, err = maxprocs.Set(maxprocs.Logger(func(str string, params ...any) { + return gomaxecs.Set(gomaxecs.WithLogger(func(str string, params ...any) { set.Logger.Debug(fmt.Sprintf(str, params)) })) } - return undoFunc(undo), err + return maxprocs.Set(maxprocs.Logger(func(str string, params ...any) { + set.Logger.Debug(fmt.Sprintf(str, params)) + })) }, func(ratio float64) (undoFunc, error) { initial, err := memlimit.SetGoMemLimitWithOpts(memlimit.WithRatio(ratio)) From f5abdc4c9a2d0190d00c9f70273e215d1f87e2e4 Mon Sep 17 00:00:00 2001 From: Romain Dauby Date: Thu, 26 Dec 2024 18:04:40 -0500 Subject: [PATCH 05/13] Add changelog --- ...k-ecs-metadata-cgroupruntimeextension.yaml | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 .chloggen/check-ecs-metadata-cgroupruntimeextension.yaml diff --git a/.chloggen/check-ecs-metadata-cgroupruntimeextension.yaml b/.chloggen/check-ecs-metadata-cgroupruntimeextension.yaml new file mode 100644 index 000000000000..6c0bdd7c0f19 --- /dev/null +++ b/.chloggen/check-ecs-metadata-cgroupruntimeextension.yaml @@ -0,0 +1,27 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: extension/cgroupruntime + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Implement ECS metadata retrieval for cgroupruntime extension. + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [36814] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: + +# If your change doesn't affect end users or the exported elements of any package, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [] \ No newline at end of file From bd1e45c2cbf9f93721c5cf54386bb10472f2ea5d Mon Sep 17 00:00:00 2001 From: Romain Dauby Date: Thu, 26 Dec 2024 22:56:09 -0500 Subject: [PATCH 06/13] Fix changelog component name --- .chloggen/check-ecs-metadata-cgroupruntimeextension.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.chloggen/check-ecs-metadata-cgroupruntimeextension.yaml b/.chloggen/check-ecs-metadata-cgroupruntimeextension.yaml index 6c0bdd7c0f19..0a8a2728ee60 100644 --- a/.chloggen/check-ecs-metadata-cgroupruntimeextension.yaml +++ b/.chloggen/check-ecs-metadata-cgroupruntimeextension.yaml @@ -4,7 +4,7 @@ change_type: enhancement # The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) -component: extension/cgroupruntime +component: cgroupruntimeextension # A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). note: Implement ECS metadata retrieval for cgroupruntime extension. From 089ae1f325a6e6565ebaa783f99225480a8cb576 Mon Sep 17 00:00:00 2001 From: Romain Dauby Date: Fri, 27 Dec 2024 11:48:05 -0500 Subject: [PATCH 07/13] Fix integration test for ECS --- .../integration_test.go | 106 ++++++++++++++---- 1 file changed, 82 insertions(+), 24 deletions(-) diff --git a/extension/cgroupruntimeextension/integration_test.go b/extension/cgroupruntimeextension/integration_test.go index 56f356e89fd7..d14fa4a06d41 100644 --- a/extension/cgroupruntimeextension/integration_test.go +++ b/extension/cgroupruntimeextension/integration_test.go @@ -9,7 +9,6 @@ package cgroupruntimeextension // import "github.com/open-telemetry/opentelemetr import ( "context" - "encoding/json" "fmt" "math" "net/http" @@ -33,6 +32,7 @@ import ( const ( defaultCgroup2Path = "/sys/fs/cgroup" + ecsMetadataUri = "ECS_CONTAINER_METADATA_URI_V4" ) // checkCgroupSystem skips the test if is not run in a cgroupv2 system @@ -66,19 +66,24 @@ func cgroupMaxCPU(filename string) (quota int64, period uint64, err error) { return quota, period, err } -func startMockECSServer() *httptest.Server { - handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - response := map[string]interface{}{ - "DockerID": "container-id", - "Limits": map[string]interface{}{ - "CPU": 2.0, - }, - } - w.Header().Set("Content-Type", "application/json") - json.NewEncoder(w).Encode(response) +func testServerECSMetadata(t *testing.T, containerCPU, taskCPU int) *httptest.Server { + t.Helper() + + mux := http.NewServeMux() + mux.HandleFunc("/", func(w http.ResponseWriter, _ *http.Request) { + _, err := w.Write([]byte(fmt.Sprintf(`{"Limits":{"CPU":%d},"DockerId":"container-id"}`, containerCPU))) + assert.NoError(t, err) + }) + mux.HandleFunc("/task", func(w http.ResponseWriter, _ *http.Request) { + _, err := w.Write([]byte(fmt.Sprintf( + `{"Containers":[{"DockerId":"container-id","Limits":{"CPU":%d}}],"Limits":{"CPU":%d}}`, + containerCPU, + taskCPU, + ))) + assert.NoError(t, err) }) - return httptest.NewServer(handler) + return httptest.NewServer(mux) } func TestCgroupV2SudoIntegration(t *testing.T) { @@ -164,10 +169,11 @@ func TestCgroupV2SudoIntegration(t *testing.T) { expectedGoMemLimit: 13421772, }, { - name: "running on AWS ECS with 90% of max cgroup memory and 2 GOMAXPROCS", - cgroupCpuQuota: pointerInt64(-1), + name: "AWS ECS 90% the max cgroup memory and 12 GOMAXPROCS", + cgroupCpuQuota: pointerInt64(100000), cgroupCpuPeriod: 8000, - cgroupMaxMemory: 134217728, // 128 MB + // 128 Mb + cgroupMaxMemory: 134217728, config: &Config{ GoMaxProcs: GoMaxProcsConfig{ Enabled: true, @@ -177,8 +183,54 @@ func TestCgroupV2SudoIntegration(t *testing.T) { Ratio: 0.9, }, }, - expectedGoMaxProcs: 22, - expectedGoMemLimit: 120795955, // 134217728 * 0.9 + // 100000 / 8000 + expectedGoMaxProcs: 12, + // 134217728 * 0.9 + expectedGoMemLimit: 120795955, + setECSMetadataURI: true, + }, + { + name: "AWS ECS 50% of the max cgroup memory and 1 GOMAXPROCS", + cgroupCpuQuota: pointerInt64(100000), + cgroupCpuPeriod: 100000, + // 128 Mb + cgroupMaxMemory: 134217728, + config: &Config{ + GoMaxProcs: GoMaxProcsConfig{ + Enabled: true, + }, + GoMemLimit: GoMemLimitConfig{ + Enabled: true, + Ratio: 0.5, + }, + }, + // 100000 / 100000 + expectedGoMaxProcs: 1, + // 134217728 * 0.5 + expectedGoMemLimit: 67108864, + setECSMetadataURI: true, + }, + { + name: "AWS ECS 10% of the max cgroup memory, max cpu, default GOMAXPROCS", + cgroupCpuQuota: nil, + cgroupCpuPeriod: 100000, + // 128 Mb + cgroupMaxMemory: 134217728, + config: &Config{ + GoMaxProcs: GoMaxProcsConfig{ + Enabled: true, + }, + GoMemLimit: GoMemLimitConfig{ + Enabled: true, + Ratio: 0.1, + }, + }, + // GOMAXPROCS is set to the value of `cpu.max / cpu.period` + // If cpu.max is set to max, GOMAXPROCS should not be + // modified + expectedGoMaxProcs: runtime.GOMAXPROCS(-1), + // 134217728 * 0.1 + expectedGoMemLimit: 13421772, setECSMetadataURI: true, }, } @@ -235,12 +287,25 @@ func TestCgroupV2SudoIntegration(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { + // if running in ECS environment, set the ECS metedata URI environment variable + // to get the Cgroup CPU quota from the httptest server + cleanECS := func() {} + if test.setECSMetadataURI { + server := testServerECSMetadata(t, test.expectedGoMaxProcs*1024, test.expectedGoMaxProcs*1024) + os.Setenv(ecsMetadataUri, server.URL) + cleanECS = func() { + server.Close() + os.Unsetenv(ecsMetadataUri) + } + } + // restore startup cgroup initial resource values t.Cleanup(func() { debug.SetMemoryLimit(initialGoMem) runtime.GOMAXPROCS(initialGoProcs) memoryCgroupCleanUp() cpuCgroupCleanUp() + cleanECS() }) err = manager.Update(&cgroup2.Resources{ @@ -257,13 +322,6 @@ func TestCgroupV2SudoIntegration(t *testing.T) { }) require.NoError(t, err) - if test.setECSMetadataURI { - server := startMockECSServer() - defer server.Close() - os.Setenv("ECS_CONTAINER_METADATA_URI_V4", server.URL) - defer os.Unsetenv("ECS_CONTAINER_METADATA_URI_V4") - } - factory := NewFactory() ctx := context.Background() extension, err := factory.Create(ctx, extensiontest.NewNopSettings(), test.config) From e8fd112bd5148648e43258b053c8ac7f7137888f Mon Sep 17 00:00:00 2001 From: Romain Dauby Date: Mon, 30 Dec 2024 13:20:38 -0500 Subject: [PATCH 08/13] Contributing guide for running local integration tests --- .../cgroupruntimeextension/CONTRIBUTING.md | 26 +++++++++++++++++++ extension/cgroupruntimeextension/README.md | 4 +++ 2 files changed, 30 insertions(+) create mode 100644 extension/cgroupruntimeextension/CONTRIBUTING.md diff --git a/extension/cgroupruntimeextension/CONTRIBUTING.md b/extension/cgroupruntimeextension/CONTRIBUTING.md new file mode 100644 index 000000000000..b2030ac8c36b --- /dev/null +++ b/extension/cgroupruntimeextension/CONTRIBUTING.md @@ -0,0 +1,26 @@ +# Contributing to the Cgroup Go runtime extension + +In order to contribute to this extension, it might be useful to have a working local setup. + +## Testing + +To run the integration tests locally for this extension, you can follow theses steps in a Linux environment. + +Inside the extension folder, start a privileged docker container and share the code with the container + +```bash +cd extension/cgroupruntimeextension +docker run -ti --privileged --cgroupns=host -v $(pwd):/workspace -w /workspace debian:bookworm-slim +``` + +Install Go and gcc to run the integration test + +```bash +apt update && apt install -y wget sudo gcc && wget https://go.dev/dl/go1.23.4.linux-amd64.tar.gz && tar -C /usr/local -xzf go1.23.4.linux-amd64.tar.gz && export PATH=$PATH:/usr/local/go/bin && go version && rm go1.23.4.linux-amd64.tar.gz +``` + +Run the integration test + +```bash +CGO_ENABLED=1 go test -v -exec sudo -race -timeout 360s -parallel 4 -tags=integration,"" +``` diff --git a/extension/cgroupruntimeextension/README.md b/extension/cgroupruntimeextension/README.md index 995be9cf314c..8af1a28d88b1 100644 --- a/extension/cgroupruntimeextension/README.md +++ b/extension/cgroupruntimeextension/README.md @@ -40,3 +40,7 @@ extension: enabled: true ratio: 0.8 ``` + +## Contributing + +See [CONTRIBUTING.md](CONTRIBUTING.md) for information on how to contribute to this extension. From c68fb693d2fd7a177df81901f95fff8a0343e66b Mon Sep 17 00:00:00 2001 From: Romain Dauby Date: Tue, 31 Dec 2024 09:23:38 -0500 Subject: [PATCH 09/13] Update extension/cgroupruntimeextension/CONTRIBUTING.md Co-authored-by: Roger Coll --- extension/cgroupruntimeextension/CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extension/cgroupruntimeextension/CONTRIBUTING.md b/extension/cgroupruntimeextension/CONTRIBUTING.md index b2030ac8c36b..be7ec1d88a7c 100644 --- a/extension/cgroupruntimeextension/CONTRIBUTING.md +++ b/extension/cgroupruntimeextension/CONTRIBUTING.md @@ -13,7 +13,7 @@ cd extension/cgroupruntimeextension docker run -ti --privileged --cgroupns=host -v $(pwd):/workspace -w /workspace debian:bookworm-slim ``` -Install Go and gcc to run the integration test +Install the [Go version](https://go.dev/dl/) specified in the extension's [go.mod](./go.mod) and the GCC compiler to run the integration test. The following is an example command for Go `1.23.4` in and `amd64` system: ```bash apt update && apt install -y wget sudo gcc && wget https://go.dev/dl/go1.23.4.linux-amd64.tar.gz && tar -C /usr/local -xzf go1.23.4.linux-amd64.tar.gz && export PATH=$PATH:/usr/local/go/bin && go version && rm go1.23.4.linux-amd64.tar.gz From 374485ae8ec695f2e8b0781b1d066cb060286576 Mon Sep 17 00:00:00 2001 From: Romain Dauby Date: Tue, 31 Dec 2024 09:27:46 -0500 Subject: [PATCH 10/13] Notice for Linux systemd cgroupv2 --- extension/cgroupruntimeextension/CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extension/cgroupruntimeextension/CONTRIBUTING.md b/extension/cgroupruntimeextension/CONTRIBUTING.md index be7ec1d88a7c..a5fbbc80db1b 100644 --- a/extension/cgroupruntimeextension/CONTRIBUTING.md +++ b/extension/cgroupruntimeextension/CONTRIBUTING.md @@ -4,7 +4,7 @@ In order to contribute to this extension, it might be useful to have a working l ## Testing -To run the integration tests locally for this extension, you can follow theses steps in a Linux environment. +Some Linux distributions don't run systemd under cgroupv2, to run the integration tests locally for this extension you can follow these steps. Inside the extension folder, start a privileged docker container and share the code with the container From 9ed2e39c749e7a899c09e9b2013e0775606a2a06 Mon Sep 17 00:00:00 2001 From: Romain Dauby Date: Tue, 31 Dec 2024 10:31:15 -0500 Subject: [PATCH 11/13] Dedicated ECS integration test --- .../integration_test.go | 282 +++++++++++------- 1 file changed, 176 insertions(+), 106 deletions(-) diff --git a/extension/cgroupruntimeextension/integration_test.go b/extension/cgroupruntimeextension/integration_test.go index d14fa4a06d41..8e5326bc50fe 100644 --- a/extension/cgroupruntimeextension/integration_test.go +++ b/extension/cgroupruntimeextension/integration_test.go @@ -49,7 +49,40 @@ func checkCgroupSystem(tb testing.TB) { } } -// cgroupMaxCPU returns the CPU max definition for a given cgroup slice path +func pointerInt64(val int64) *int64 { + return &val +} + +func pointerUint64(uval uint64) *uint64 { + return &uval +} + +// setupMemoryCgroupCleanUp returns a cleanup function that restores the cgroup's max memory to its initial value +func setupMemoryCgroupCleanUp(t *testing.T, manager *cgroup2.Manager, cgroupPath string) func() { + stats, err := manager.Stat() + require.NoError(t, err) + + initialMaxMemory := stats.GetMemory().GetUsageLimit() + memoryCgroupCleanUp := func() { + err = manager.Update(&cgroup2.Resources{ + Memory: &cgroup2.Memory{ + Max: pointerInt64(int64(initialMaxMemory)), + }, + }) + assert.NoError(t, err) + } + + if initialMaxMemory == math.MaxUint64 { + // fallback solution to set cgroup's max memory to "max" + memoryCgroupCleanUp = func() { + err = os.WriteFile(path.Join(defaultCgroup2Path, cgroupPath, "memory.max"), []byte("max"), 0o600) + assert.NoError(t, err) + } + } + return memoryCgroupCleanUp +} + +// cgroupMaxCpu returns the CPU max definition for a given cgroup slice path // File format: cpu_quote cpu_period func cgroupMaxCPU(filename string) (quota int64, period uint64, err error) { out, err := os.ReadFile(filepath.Join(defaultCgroup2Path, filename, "cpu.max")) @@ -66,34 +99,19 @@ func cgroupMaxCPU(filename string) (quota int64, period uint64, err error) { return quota, period, err } -func testServerECSMetadata(t *testing.T, containerCPU, taskCPU int) *httptest.Server { - t.Helper() - - mux := http.NewServeMux() - mux.HandleFunc("/", func(w http.ResponseWriter, _ *http.Request) { - _, err := w.Write([]byte(fmt.Sprintf(`{"Limits":{"CPU":%d},"DockerId":"container-id"}`, containerCPU))) - assert.NoError(t, err) - }) - mux.HandleFunc("/task", func(w http.ResponseWriter, _ *http.Request) { - _, err := w.Write([]byte(fmt.Sprintf( - `{"Containers":[{"DockerId":"container-id","Limits":{"CPU":%d}}],"Limits":{"CPU":%d}}`, - containerCPU, - taskCPU, - ))) - assert.NoError(t, err) - }) +// startExtension starts the extension with the given config +func startExtension(t *testing.T, config *Config) { + factory := NewFactory() + ctx := context.Background() + extension, err := factory.Create(ctx, extensiontest.NewNopSettings(), config) + require.NoError(t, err) - return httptest.NewServer(mux) + err = extension.Start(ctx, componenttest.NewNopHost()) + require.NoError(t, err) } func TestCgroupV2SudoIntegration(t *testing.T) { checkCgroupSystem(t) - pointerInt64 := func(val int64) *int64 { - return &val - } - pointerUint64 := func(uval uint64) *uint64 { - return &uval - } tests := []struct { name string @@ -168,10 +186,107 @@ func TestCgroupV2SudoIntegration(t *testing.T) { // 134217728 * 0.1 expectedGoMemLimit: 13421772, }, + } + + cgroupPath, err := cgroup2.PidGroupPath(os.Getpid()) + assert.NoError(t, err) + manager, err := cgroup2.Load(cgroupPath) + assert.NoError(t, err) + + // Startup resource values + memoryCgroupCleanUp := setupMemoryCgroupCleanUp(t, manager, cgroupPath) + + initialCPUQuota, initialCPUPeriod, err := cgroupMaxCPU(cgroupPath) + require.NoError(t, err) + cpuCgroupCleanUp := func() { + fmt.Println(initialCPUQuota) + err = manager.Update(&cgroup2.Resources{ + CPU: &cgroup2.CPU{ + Max: cgroup2.NewCPUMax(pointerInt64(initialCPUQuota), pointerUint64(initialCPUPeriod)), + }, + }) + assert.NoError(t, err) + } + + if initialCPUQuota == math.MaxInt64 { + // fallback solution to set cgroup's max cpu to "max" + cpuCgroupCleanUp = func() { + err = os.WriteFile(path.Join(defaultCgroup2Path, cgroupPath, "cpu.max"), []byte("max"), 0o600) + assert.NoError(t, err) + } + } + + initialGoMem := debug.SetMemoryLimit(-1) + initialGoProcs := runtime.GOMAXPROCS(-1) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + // restore startup cgroup initial resource values + t.Cleanup(func() { + debug.SetMemoryLimit(initialGoMem) + runtime.GOMAXPROCS(initialGoProcs) + memoryCgroupCleanUp() + cpuCgroupCleanUp() + }) + + err = manager.Update(&cgroup2.Resources{ + Memory: &cgroup2.Memory{ + // Default max memory must be + // overwritten + // to automemlimit change the GOMEMLIMIT + // value + Max: pointerInt64(test.cgroupMaxMemory), + }, + CPU: &cgroup2.CPU{ + Max: cgroup2.NewCPUMax(test.cgroupCPUQuota, pointerUint64(test.cgroupCPUPeriod)), + }, + }) + require.NoError(t, err) + + startExtension(t, test.config) + + assert.Equal(t, test.expectedGoMaxProcs, runtime.GOMAXPROCS(-1)) + assert.Equal(t, test.expectedGoMemLimit, debug.SetMemoryLimit(-1)) + }) + } +} + +func testServerECSMetadata(t *testing.T, containerCPU, taskCPU int) *httptest.Server { + t.Helper() + + mux := http.NewServeMux() + mux.HandleFunc("/", func(w http.ResponseWriter, _ *http.Request) { + _, err := w.Write([]byte(fmt.Sprintf(`{"Limits":{"CPU":%d},"DockerId":"container-id"}`, containerCPU))) + assert.NoError(t, err) + }) + mux.HandleFunc("/task", func(w http.ResponseWriter, _ *http.Request) { + _, err := w.Write([]byte(fmt.Sprintf( + `{"Containers":[{"DockerId":"container-id","Limits":{"CPU":%d}}],"Limits":{"CPU":%d}}`, + containerCPU, + taskCPU, + ))) + assert.NoError(t, err) + }) + + return httptest.NewServer(mux) +} + +func TestECSCgroupV2SudoIntegration(t *testing.T) { + checkCgroupSystem(t) + + tests := []struct { + name string + containerCPU int + taskCPU int + cgroupMaxMemory int64 + config *Config + expectedGoMaxProcs int + expectedGoMemLimit int64 + }{ { - name: "AWS ECS 90% the max cgroup memory and 12 GOMAXPROCS", - cgroupCpuQuota: pointerInt64(100000), - cgroupCpuPeriod: 8000, + name: "90% the max cgroup memory and 4 GOMAXPROCS w/ 4096 container cpu 16 task cpu", + containerCPU: 4096, + taskCPU: 16, // 128 Mb cgroupMaxMemory: 134217728, config: &Config{ @@ -183,16 +298,33 @@ func TestCgroupV2SudoIntegration(t *testing.T) { Ratio: 0.9, }, }, - // 100000 / 8000 - expectedGoMaxProcs: 12, + expectedGoMaxProcs: 4, // 134217728 * 0.9 expectedGoMemLimit: 120795955, - setECSMetadataURI: true, }, { - name: "AWS ECS 50% of the max cgroup memory and 1 GOMAXPROCS", - cgroupCpuQuota: pointerInt64(100000), - cgroupCpuPeriod: 100000, + name: "50% of the max cgroup memory and 1 GOMAXPROCS w/ 2048 container cpu 2 task cpu", + containerCPU: 2048, + taskCPU: 2, + // 128 Mb + cgroupMaxMemory: 134217728, + config: &Config{ + GoMaxProcs: GoMaxProcsConfig{ + Enabled: true, + }, + GoMemLimit: GoMemLimitConfig{ + Enabled: true, + Ratio: 0.5, + }, + }, + expectedGoMaxProcs: 2, + // 134217728 * 0.5 + expectedGoMemLimit: 67108864, + }, + { + name: "50% of the max cgroup memory and 1 GOMAXPROCS w/ 1024 container cpu 4 task cpu", + containerCPU: 1024, + taskCPU: 4, // 128 Mb cgroupMaxMemory: 134217728, config: &Config{ @@ -204,16 +336,14 @@ func TestCgroupV2SudoIntegration(t *testing.T) { Ratio: 0.5, }, }, - // 100000 / 100000 expectedGoMaxProcs: 1, // 134217728 * 0.5 expectedGoMemLimit: 67108864, - setECSMetadataURI: true, }, { - name: "AWS ECS 10% of the max cgroup memory, max cpu, default GOMAXPROCS", - cgroupCpuQuota: nil, - cgroupCpuPeriod: 100000, + name: "10% of the max cgroup memory and 4 GOMAXPROCS w/ 4096 container cpu 0 task cpu", + containerCPU: 4096, + taskCPU: 0, // 128 Mb cgroupMaxMemory: 134217728, config: &Config{ @@ -225,13 +355,9 @@ func TestCgroupV2SudoIntegration(t *testing.T) { Ratio: 0.1, }, }, - // GOMAXPROCS is set to the value of `cpu.max / cpu.period` - // If cpu.max is set to max, GOMAXPROCS should not be - // modified - expectedGoMaxProcs: runtime.GOMAXPROCS(-1), + expectedGoMaxProcs: 4, // 134217728 * 0.1 expectedGoMemLimit: 13421772, - setECSMetadataURI: true, }, } @@ -240,72 +366,25 @@ func TestCgroupV2SudoIntegration(t *testing.T) { manager, err := cgroup2.Load(cgroupPath) assert.NoError(t, err) - stats, err := manager.Stat() - require.NoError(t, err) - // Startup resource values - initialMaxMemory := stats.GetMemory().GetUsageLimit() - memoryCgroupCleanUp := func() { - err = manager.Update(&cgroup2.Resources{ - Memory: &cgroup2.Memory{ - Max: pointerInt64(int64(initialMaxMemory)), - }, - }) - assert.NoError(t, err) - } - - if initialMaxMemory == math.MaxUint64 { - // fallback solution to set cgroup's max memory to "max" - memoryCgroupCleanUp = func() { - err = os.WriteFile(path.Join(defaultCgroup2Path, cgroupPath, "memory.max"), []byte("max"), 0o600) - assert.NoError(t, err) - } - } - - initialCPUQuota, initialCPUPeriod, err := cgroupMaxCPU(cgroupPath) - require.NoError(t, err) - cpuCgroupCleanUp := func() { - fmt.Println(initialCPUQuota) - err = manager.Update(&cgroup2.Resources{ - CPU: &cgroup2.CPU{ - Max: cgroup2.NewCPUMax(pointerInt64(initialCPUQuota), pointerUint64(initialCPUPeriod)), - }, - }) - assert.NoError(t, err) - } - - if initialCPUQuota == math.MaxInt64 { - // fallback solution to set cgroup's max cpu to "max" - cpuCgroupCleanUp = func() { - err = os.WriteFile(path.Join(defaultCgroup2Path, cgroupPath, "cpu.max"), []byte("max"), 0o600) - assert.NoError(t, err) - } - } + memoryCgroupCleanUp := setupMemoryCgroupCleanUp(t, manager, cgroupPath) initialGoMem := debug.SetMemoryLimit(-1) initialGoProcs := runtime.GOMAXPROCS(-1) for _, test := range tests { t.Run(test.name, func(t *testing.T) { - // if running in ECS environment, set the ECS metedata URI environment variable + // running in ECS environment, set the ECS metedata URI environment variable // to get the Cgroup CPU quota from the httptest server - cleanECS := func() {} - if test.setECSMetadataURI { - server := testServerECSMetadata(t, test.expectedGoMaxProcs*1024, test.expectedGoMaxProcs*1024) - os.Setenv(ecsMetadataUri, server.URL) - cleanECS = func() { - server.Close() - os.Unsetenv(ecsMetadataUri) - } - } - + server := testServerECSMetadata(t, test.containerCPU, test.taskCPU) + os.Setenv(ecsMetadataUri, server.URL) // restore startup cgroup initial resource values t.Cleanup(func() { debug.SetMemoryLimit(initialGoMem) runtime.GOMAXPROCS(initialGoProcs) memoryCgroupCleanUp() - cpuCgroupCleanUp() - cleanECS() + server.Close() + os.Unsetenv(ecsMetadataUri) }) err = manager.Update(&cgroup2.Resources{ @@ -316,19 +395,10 @@ func TestCgroupV2SudoIntegration(t *testing.T) { // value Max: pointerInt64(test.cgroupMaxMemory), }, - CPU: &cgroup2.CPU{ - Max: cgroup2.NewCPUMax(test.cgroupCPUQuota, pointerUint64(test.cgroupCPUPeriod)), - }, }) require.NoError(t, err) - factory := NewFactory() - ctx := context.Background() - extension, err := factory.Create(ctx, extensiontest.NewNopSettings(), test.config) - require.NoError(t, err) - - err = extension.Start(ctx, componenttest.NewNopHost()) - require.NoError(t, err) + startExtension(t, test.config) assert.Equal(t, test.expectedGoMaxProcs, runtime.GOMAXPROCS(-1)) assert.Equal(t, test.expectedGoMemLimit, debug.SetMemoryLimit(-1)) From 18e1f053cb5d163f9571e72dd28479b16d5e75b8 Mon Sep 17 00:00:00 2001 From: Romain Dauby Date: Tue, 31 Dec 2024 10:39:41 -0500 Subject: [PATCH 12/13] Remove unused test property --- extension/cgroupruntimeextension/integration_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/extension/cgroupruntimeextension/integration_test.go b/extension/cgroupruntimeextension/integration_test.go index 8e5326bc50fe..e23d83f7185d 100644 --- a/extension/cgroupruntimeextension/integration_test.go +++ b/extension/cgroupruntimeextension/integration_test.go @@ -122,7 +122,6 @@ func TestCgroupV2SudoIntegration(t *testing.T) { config *Config expectedGoMaxProcs int expectedGoMemLimit int64 - setECSMetadataURI bool }{ { name: "90% the max cgroup memory and 12 GOMAXPROCS", From e8f57e77066f880cd7f46a7a80c93a5bdfe233d7 Mon Sep 17 00:00:00 2001 From: Romain Dauby Date: Tue, 7 Jan 2025 11:49:18 -0500 Subject: [PATCH 13/13] Fix CI lint --- extension/cgroupruntimeextension/integration_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/extension/cgroupruntimeextension/integration_test.go b/extension/cgroupruntimeextension/integration_test.go index e23d83f7185d..29032012a295 100644 --- a/extension/cgroupruntimeextension/integration_test.go +++ b/extension/cgroupruntimeextension/integration_test.go @@ -32,7 +32,7 @@ import ( const ( defaultCgroup2Path = "/sys/fs/cgroup" - ecsMetadataUri = "ECS_CONTAINER_METADATA_URI_V4" + ecsMetadataURI = "ECS_CONTAINER_METADATA_URI_V4" ) // checkCgroupSystem skips the test if is not run in a cgroupv2 system @@ -376,14 +376,14 @@ func TestECSCgroupV2SudoIntegration(t *testing.T) { // running in ECS environment, set the ECS metedata URI environment variable // to get the Cgroup CPU quota from the httptest server server := testServerECSMetadata(t, test.containerCPU, test.taskCPU) - os.Setenv(ecsMetadataUri, server.URL) + t.Setenv(ecsMetadataURI, server.URL) // restore startup cgroup initial resource values t.Cleanup(func() { debug.SetMemoryLimit(initialGoMem) runtime.GOMAXPROCS(initialGoProcs) memoryCgroupCleanUp() server.Close() - os.Unsetenv(ecsMetadataUri) + os.Unsetenv(ecsMetadataURI) }) err = manager.Update(&cgroup2.Resources{