From 7441ae83ade49f9d9ecc08c3556aac5dec0c351f Mon Sep 17 00:00:00 2001 From: Ziwen Ning Date: Thu, 28 Sep 2023 22:30:38 -0700 Subject: [PATCH] ci: add benchmarking Signed-off-by: Ziwen Ning --- .github/workflows/benchmark.yaml | 144 +++++++++++++++++++++++++++ .github/workflows/ci.yaml | 32 +----- Makefile | 20 +++- benchmark/awslogs/awslogs_test.go | 86 ++++++++++++++++ benchmark/fluentd/fluentd_test.go | 34 +++++++ benchmark/splunk/splunk_test.go | 45 +++++++++ e2e/awslogs_test.go | 74 +++++++------- e2e/common.go | 93 +++++++++++++++++ e2e/fluentd_test.go | 50 +++++----- e2e/main_test.go | 80 +-------------- e2e/splunk_test.go | 10 +- scripts/start-fluentd-local-endpoint | 19 ++++ scripts/start-localstack | 11 ++ 13 files changed, 527 insertions(+), 171 deletions(-) create mode 100644 .github/workflows/benchmark.yaml create mode 100644 benchmark/awslogs/awslogs_test.go create mode 100644 benchmark/fluentd/fluentd_test.go create mode 100644 benchmark/splunk/splunk_test.go create mode 100644 e2e/common.go create mode 100755 scripts/start-fluentd-local-endpoint create mode 100755 scripts/start-localstack diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml new file mode 100644 index 0000000..407f012 --- /dev/null +++ b/.github/workflows/benchmark.yaml @@ -0,0 +1,144 @@ +name: Benchmark +on: + push: + branches: + - main + +permissions: + # deployments permission to deploy GitHub pages website + deployments: write + # contents permission to update benchmark contents in gh-pages branch + contents: write + +jobs: + benchmark-for-awslogs: + strategy: + fail-fast: false + matrix: + os: [ ubuntu-latest ] # TODO: Add Windows e2e tests: https://github.com/aws/shim-loggers-for-containerd/issues/68 + name: Benchmark / awslogs / ${{ matrix.os }} + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-go@v4 + with: + go-version: '1.21' + cache: false + - name: Start LocalStack + shell: bash + run: scripts/start-localstack + - name: install and start containerd + shell: bash + run: sudo scripts/install-containerd + - name: start ecs local endpoint + shell: bash + run: scripts/start-ecs-local-endpoint + - name: ip forwarding # awslogs driver hardcodes "169.254.170.2" as the aws credential endpoint ip so need to forward to local endpoint + shell: bash + run: sudo scripts/ip-forwarding + - name: build + run: sudo make build + - name: Run benchmark + run: sudo -E make test-benchmark-for-awslogs | tee benchmark-for-awslogs.txt + shell: bash + - name: Store benchmark result + uses: benchmark-action/github-action-benchmark@v1 + with: + name: Benchmark for awslogs + tool: 'go' + benchmark-data-dir-path: "dev/bench/awslogs/ubuntu" + output-file-path: benchmark-for-awslogs.txt + - name: Push benchmark result + run: git push 'https://github.com/aws/shim-loggers-for-containerd.git' gh-pages:gh-pages + benchmark-for-fluentd: + strategy: + fail-fast: false + matrix: + os: [ ubuntu-latest ] # TODO: Add Windows e2e tests: https://github.com/aws/shim-loggers-for-containerd/issues/68 + name: Benchmark / fluentd / ${{ matrix.os }} + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-go@v4 + with: + go-version: '1.21' + cache: false + - name: install and start containerd + shell: bash + run: sudo scripts/install-containerd + - name: start fluentd local endpoint + shell: bash + run: scripts/start-fluentd-local-endpoint + - name: build + run: sudo make build + - name: Run benchmark + run: sudo make test-benchmark-for-fluentd | tee benchmark-for-fluentd.txt + shell: bash + - name: Store benchmark result + uses: benchmark-action/github-action-benchmark@v1 + with: + name: Benchmark for awslogs + tool: 'go' + benchmark-data-dir-path: "dev/bench/fluentd/ubuntu" + output-file-path: benchmark-for-fluentd.txt + - name: Push benchmark result + run: git push 'https://github.com/aws/shim-loggers-for-containerd.git' gh-pages:gh-pages + benchmark-for-splunk: + strategy: + fail-fast: false + matrix: + os: [ ubuntu-latest ] # TODO: Add Windows e2e tests: https://github.com/aws/shim-loggers-for-containerd/issues/68 + name: Benchmark / splunk / ${{ matrix.os }} + runs-on: ${{ matrix.os }} + permissions: + id-token: write + contents: write + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-go@v4 + with: + go-version: '1.21' + cache: false + - name: install and start containerd + shell: bash + run: sudo scripts/install-containerd + - name: start splunk local endpoint + id: splunk-local-endpoint + shell: bash + run: | + SPLUNK_PASSWORD=1234567Aa! + SPLUNK_PORT=8089 + SPLUNK_IMAGE=splunk/splunk + CONTAINER_ID=$(docker run -d -p $SPLUNK_PORT:8089 -p 8088:8088 \ + -e "SPLUNK_START_ARGS=--accept-license" \ + -e "SPLUNK_PASSWORD=$SPLUNK_PASSWORD" \ + -e "SPLUNK_HEC_TOKEN=abcd1234" \ + $SPLUNK_IMAGE) + + : # Splunk can only receives requests after becoming healthy + until [ $(docker inspect $CONTAINER_ID --format ‘{{.State.Health.Status}}’) == ‘healthy’ ]; do sleep 10s; done; + + curl -L -k -u admin:$SPLUNK_PASSWORD \ + -X POST https://localhost:$SPLUNK_PORT/services/admin/token-auth/tokens_auth \ + -d disabled=false + : # Wait for token to be enabled + sleep 5 + TOKEN_OUTPUT=$(curl -L -k -u admin:$SPLUNK_PASSWORD \ + -X POST https://localhost:$SPLUNK_PORT/services/authorization/tokens?output_mode=json \ + -d name=admin -d audience=tests) + SPLUNK_TOKEN=$(echo $TOKEN_OUTPUT | jq -r '.entry[0].content.token') + echo "SPLUNK_TOKEN=$SPLUNK_TOKEN" >> $GITHUB_OUTPUT + - name: build + run: sudo make build + - name: Run benchmark + run: sudo SPLUNK_TOKEN=${{ steps.splunk-local-endpoint.outputs.SPLUNK_TOKEN }} make test-benchmark-for-splunk | tee benchmark-for-splunk.txt + shell: bash + - name: Store benchmark result + uses: benchmark-action/github-action-benchmark@v1 + with: + name: Benchmark for splunk + tool: 'go' + benchmark-data-dir-path: "dev/bench/splunk/ubuntu" + output-file-path: benchmark-for-splunk.txt + - name: Push benchmark result + run: git push 'https://github.com/aws/shim-loggers-for-containerd.git' gh-pages:gh-pages diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 5941894..146e9a8 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -62,16 +62,9 @@ jobs: with: go-version: ${{ matrix.go }} cache: false - - name: Start LocalStack # Copy from https://docs.localstack.cloud/user-guide/ci/github-actions/ - run: | - LOCALSTACK_VERSION=2.2.0 - pip install localstack==${LOCALSTACK_VERSION} awscli-local[ver1] # install LocalStack cli and awslocal - docker pull localstack/localstack:${LOCALSTACK_VERSION} # Make sure to pull the latest version of the image - localstack start -d # Start LocalStack in the background - - echo "Waiting for LocalStack startup..." # Wait 30 seconds for the LocalStack container - localstack wait -t 30 # to become ready before timing out - echo "Startup complete" + - name: Start LocalStack + shell: bash + run: scripts/start-localstack - name: install and start containerd shell: bash run: sudo scripts/install-containerd @@ -107,24 +100,7 @@ jobs: run: sudo scripts/install-containerd - name: start fluentd local endpoint shell: bash - run: | - : # not using github action env because env map cannot be defined in terms of other vars in the map. public.ecr.aws/docker/library/fluentd:v1.16-debian-1 - FLUENTD_LOG_DIR=${GITHUB_WORKSPACE}/fluentd-logs - FLUENTD_PORT=24224 - FLUENTD_IMAGE=public.ecr.aws/docker/library/fluentd:v1.16-debian-1 - : # ECR Public anonymous pull rate quota is 1 per sec. https://docs.aws.amazon.com/AmazonECR/latest/public/public-service-quotas.html. - : # Running multiple jobs at the same time may exceed rate limit. Retry 3 times to prevent flaky behavior. - count=0 - until [ $count -ge 3 ] - do - docker pull $FLUENTD_IMAGE && break - count=$((count+1)) - [ $count -eq 3 ] && echo "Pull command failed after 3 attempts" && exit 1 - sleep 3 - done - : # Fluentd container is not using root user so need 777 to make it writable. https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#docker-container-filesystem - sudo mkdir -m 777 $FLUENTD_LOG_DIR - docker run -d -p $FLUENTD_PORT:24224 -p $FLUENTD_PORT:24224/udp -v $FLUENTD_LOG_DIR:/fluentd/log $FLUENTD_IMAGE + run: scripts/start-fluentd-local-endpoint - name: build run: sudo make build - name: test-e2e diff --git a/Makefile b/Makefile index 9f0e981..8ed3dc5 100644 --- a/Makefile +++ b/Makefile @@ -29,19 +29,31 @@ test-unit: $(SOURCES) .PHONY: test-e2e test-e2e: - go test -timeout 30m ./e2e -test.v -ginkgo.v --binary "$(AWS_CONTAINERD_LOGGERS_BINARY)" + go test -tags e2e -timeout 30m ./e2e -test.v -ginkgo.v --binary "$(AWS_CONTAINERD_LOGGERS_BINARY)" .PHONY: test-e2e-for-awslogs test-e2e-for-awslogs: - go test -timeout 30m ./e2e -test.v -ginkgo.v --binary "$(AWS_CONTAINERD_LOGGERS_BINARY)" --log-driver "awslogs" + go test -tags e2e -timeout 30m ./e2e -test.v -ginkgo.v --binary "$(AWS_CONTAINERD_LOGGERS_BINARY)" --log-driver "awslogs" .PHONY: test-e2e-for-fluentd test-e2e-for-fluentd: - go test -timeout 30m ./e2e -test.v -ginkgo.v --binary "$(AWS_CONTAINERD_LOGGERS_BINARY)" --log-driver "fluentd" + go test -tags e2e -timeout 30m ./e2e -test.v -ginkgo.v --binary "$(AWS_CONTAINERD_LOGGERS_BINARY)" --log-driver "fluentd" .PHONY: test-e2e-for-splunk test-e2e-for-splunk: - go test -timeout 30m ./e2e -test.v -ginkgo.v --binary "$(AWS_CONTAINERD_LOGGERS_BINARY)" --log-driver "splunk" --splunk-token ${SPLUNK_TOKEN} + go test -tags e2e -timeout 30m ./e2e -test.v -ginkgo.v --binary "$(AWS_CONTAINERD_LOGGERS_BINARY)" --log-driver "splunk" --splunk-token ${SPLUNK_TOKEN} + +.PHONY: test-benchmark-for-awslogs +test-benchmark-for-awslogs: + cd benchmark/awslogs && go test -tags benchmark,e2e -bench=Awslogs -benchmem --binary "$(AWS_CONTAINERD_LOGGERS_BINARY)" + +.PHONY: test-benchmark-for-fluentd +test-benchmark-for-fluentd: + cd benchmark/fluentd && go test -tags benchmark,e2e -bench=Fluentd -benchmem --binary "$(AWS_CONTAINERD_LOGGERS_BINARY)" + +.PHONY: test-benchmark-for-splunk +test-benchmark-for-splunk: + cd benchmark/splunk && go test -tags benchmark,e2e -bench=Splunk -benchmem --binary "$(AWS_CONTAINERD_LOGGERS_BINARY)" --splunk-token ${SPLUNK_TOKEN} .PHONY: coverage coverage: diff --git a/benchmark/awslogs/awslogs_test.go b/benchmark/awslogs/awslogs_test.go new file mode 100644 index 0000000..1639c04 --- /dev/null +++ b/benchmark/awslogs/awslogs_test.go @@ -0,0 +1,86 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +//go:build benchmark + +package awslogs + +import ( + "context" + "flag" + "strings" + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs" + "github.com/aws/shim-loggers-for-containerd/e2e" + "github.com/containerd/containerd/cio" +) + +const ( + awslogsCredentialsEndpointKey = "--awslogs-credentials-endpoint" //nolint:gosec // not credentials + awslogsRegionKey = "--awslogs-region" + awslogsStreamKey = "--awslogs-stream" + awslogsGroupKey = "--awslogs-group" + awslogsEndpointKey = "--awslogs-endpoint" + testEcsLocalEndpointPort = "51679" + testAwslogsCredentialEndpoint = ":" + testEcsLocalEndpointPort + "/creds" + testAwslogsRegion = "us-east-1" + testAwslogsStream = "test-stream" + nonExistentAwslogsStream = "non-existent-stream" + testAwslogsGroup = "test-shim-logger" + testAwslogsEndpoint = "http://localhost.localstack.cloud:4566" // Recommended endpoint: +) + +var ( + // Binary is the path the binary of the shim loggers for containerd. + Binary = flag.String("binary", "", "the binary of shim loggers for containerd") +) + +func BenchmarkAwslogs(b *testing.B) { + var cwClient *cloudwatchlogs.Client + // Reference to set up Go client for aws local stack: https://docs.localstack.cloud/user-guide/integrations/sdks/go/. + customResolver := aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...interface{}) (aws.Endpoint, error) { + return aws.Endpoint{ + PartitionID: "aws", + URL: testAwslogsEndpoint, + SigningRegion: testAwslogsRegion, + }, nil + }) + cfg, err := config.LoadDefaultConfig(context.TODO(), config.WithRegion(testAwslogsRegion), + config.WithEndpointResolverWithOptions(customResolver)) + if err != nil { + b.Fatal(err) + } + cwClient = cloudwatchlogs.NewFromConfig(cfg) + _, err = cwClient.CreateLogGroup(context.TODO(), &cloudwatchlogs.CreateLogGroupInput{ + LogGroupName: aws.String(testAwslogsGroup), + }) + if err != nil { + b.Fatal(err) + } + _, err = cwClient.CreateLogStream(context.TODO(), &cloudwatchlogs.CreateLogStreamInput{ + LogGroupName: aws.String(testAwslogsGroup), + LogStreamName: aws.String(testAwslogsStream), + }) + if err != nil { + b.Fatal(err) + } + testLog := strings.Repeat("a", 1024) + args := map[string]string{ + e2e.LogDriverTypeKey: e2e.AwslogsDriverName, + e2e.ContainerIDKey: e2e.TestContainerID, + e2e.ContainerNameKey: e2e.TestContainerName, + awslogsCredentialsEndpointKey: testAwslogsCredentialEndpoint, + awslogsRegionKey: testAwslogsRegion, + awslogsGroupKey: testAwslogsGroup, + awslogsStreamKey: nonExistentAwslogsStream, + awslogsEndpointKey: testAwslogsEndpoint, + } + creator := cio.BinaryIO(*Binary, args) + err = e2e.SendTestLogByContainerd(creator, testLog) + if err != nil { + b.Fatal(err) + } +} diff --git a/benchmark/fluentd/fluentd_test.go b/benchmark/fluentd/fluentd_test.go new file mode 100644 index 0000000..f693d24 --- /dev/null +++ b/benchmark/fluentd/fluentd_test.go @@ -0,0 +1,34 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +//go:build benchmark + +package fluentd + +import ( + "flag" + "strings" + "testing" + + "github.com/aws/shim-loggers-for-containerd/e2e" + "github.com/containerd/containerd/cio" +) + +var ( + // Binary is the path the binary of the shim loggers for containerd. + Binary = flag.String("binary", "", "the binary of shim loggers for containerd") +) + +func BenchmarkFluentd(b *testing.B) { + testLog := strings.Repeat("a", 1024) + args := map[string]string{ + e2e.LogDriverTypeKey: e2e.FluentdDriverName, + e2e.ContainerIDKey: e2e.TestContainerID, + e2e.ContainerNameKey: e2e.TestContainerName, + } + creator := cio.BinaryIO(*Binary, args) + err := e2e.SendTestLogByContainerd(creator, testLog) + if err != nil { + b.Fatal(err) + } +} diff --git a/benchmark/splunk/splunk_test.go b/benchmark/splunk/splunk_test.go new file mode 100644 index 0000000..ed6fd64 --- /dev/null +++ b/benchmark/splunk/splunk_test.go @@ -0,0 +1,45 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +//go:build benchmark + +package splunk + +import ( + "flag" + "strings" + "testing" + + "github.com/aws/shim-loggers-for-containerd/e2e" + "github.com/containerd/containerd/cio" +) + +const ( + splunkTokenKey = "--splunk-token" //nolint:gosec // no real credential + splunkURLkey = "--splunk-url" + splunkInsecureskipverifyKey = "--splunk-insecureskipverify" + testSplunkURL = "https://localhost:8089" +) + +var ( + // Binary is the path the binary of the shim loggers for containerd. + Binary = flag.String("binary", "", "the binary of shim loggers for containerd") + SplunkToken = flag.String("splunk-token", "", "the token to access Splunk") +) + +func BenchmarkSplunk(b *testing.B) { + testLog := strings.Repeat("a", 1024) + args := map[string]string{ + e2e.LogDriverTypeKey: e2e.SplunkDriverName, + e2e.ContainerIDKey: e2e.TestContainerID, + e2e.ContainerNameKey: e2e.TestContainerName, + splunkTokenKey: *SplunkToken, + splunkURLkey: testSplunkURL, + splunkInsecureskipverifyKey: "true", + } + creator := cio.BinaryIO(*Binary, args) + err := e2e.SendTestLogByContainerd(creator, testLog) + if err != nil { + b.Fatal(err) + } +} diff --git a/e2e/awslogs_test.go b/e2e/awslogs_test.go index 827eaa4..907c0fc 100644 --- a/e2e/awslogs_test.go +++ b/e2e/awslogs_test.go @@ -1,6 +1,8 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 +//go:build e2e + package e2e import ( @@ -79,9 +81,9 @@ var testAwslogs = func() { "when the configs are default", func() { testLog := testLogPrefix + uuid.New().String() args := map[string]string{ - logDriverTypeKey: awslogsDriverName, - containerIDKey: testContainerID, - containerNameKey: testContainerName, + LogDriverTypeKey: AwslogsDriverName, + ContainerIDKey: TestContainerID, + ContainerNameKey: TestContainerName, awslogsCredentialsEndpointKey: testAwslogsCredentialEndpoint, awslogsRegionKey: testAwslogsRegion, awslogsGroupKey: testAwslogsGroup, @@ -89,7 +91,7 @@ var testAwslogs = func() { awslogsEndpointKey: testAwslogsEndpoint, } creator := cio.BinaryIO(*Binary, args) - err := sendTestLogByContainerd(creator, testLog) + err := SendTestLogByContainerd(creator, testLog) gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) err = validateTestLogsInAwslogs(cwClient, testAwslogsGroup, nonExistentAwslogsStream, []string{testLog}) gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) @@ -101,9 +103,9 @@ var testAwslogs = func() { thirdLineLog := fmt.Sprintf("%s %s", testAwslogsMultilinePattern, testLogPrefix+uuid.New().String()) fourthLineLog := fmt.Sprintf("%s %s", testAwslogsMultilinePattern, testLogPrefix+uuid.New().String()) args := map[string]string{ - logDriverTypeKey: awslogsDriverName, - containerIDKey: testContainerID, - containerNameKey: testContainerName, + LogDriverTypeKey: AwslogsDriverName, + ContainerIDKey: TestContainerID, + ContainerNameKey: TestContainerName, awslogsCredentialsEndpointKey: testAwslogsCredentialEndpoint, awslogsRegionKey: testAwslogsRegion, awslogsGroupKey: testAwslogsGroup, @@ -115,7 +117,7 @@ var testAwslogs = func() { creator := cio.BinaryIO(*Binary, args) // The last matched line cannot be logged with multiline pattern. Append a pattern for now. // TODO: Investigate and fix. https://github.com/aws/shim-loggers-for-containerd/issues/78 - err := sendTestLogByContainerd(creator, fmt.Sprintf("%s\n%s\n%s\n%s\n%s", firstLineLog, + err := SendTestLogByContainerd(creator, fmt.Sprintf("%s\n%s\n%s\n%s\n%s", firstLineLog, secondLineLog, thirdLineLog, fourthLineLog, "[May 01, 2017 19:00:05]")) gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) err = validateTestLogsInAwslogs(cwClient, testAwslogsGroup, nonExistentAwslogsStream, []string{fmt.Sprintf("%s\n", firstLineLog), @@ -127,9 +129,9 @@ var testAwslogs = func() { firstLineLog := fmt.Sprintf("%s %s", testAwslogsMultilinePattern, testLogPrefix+uuid.New().String()) secondLineLog := fmt.Sprintf("%s %s", testAwslogsMultilinePattern, testLogPrefix+uuid.New().String()) args := map[string]string{ - logDriverTypeKey: awslogsDriverName, - containerIDKey: testContainerID, - containerNameKey: testContainerName, + LogDriverTypeKey: AwslogsDriverName, + ContainerIDKey: TestContainerID, + ContainerNameKey: TestContainerName, awslogsCredentialsEndpointKey: testAwslogsCredentialEndpoint, awslogsRegionKey: testAwslogsRegion, awslogsGroupKey: testAwslogsGroup, @@ -138,7 +140,7 @@ var testAwslogs = func() { awslogsMultilinePatternKey: "^" + testAwslogsMultilinePattern, } creator := cio.BinaryIO(*Binary, args) - err := sendTestLogByContainerd(creator, fmt.Sprintf("%s\n%s\n%s", firstLineLog, secondLineLog, testAwslogsMultilinePattern)) + err := SendTestLogByContainerd(creator, fmt.Sprintf("%s\n%s\n%s", firstLineLog, secondLineLog, testAwslogsMultilinePattern)) gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) err = validateTestLogsInAwslogs(cwClient, testAwslogsGroup, nonExistentAwslogsStream, []string{fmt.Sprintf("%s\n", firstLineLog), fmt.Sprintf("%s\n", secondLineLog)}) @@ -150,9 +152,9 @@ var testAwslogs = func() { firstLineLog := "[May 01, 2017 19:00:01] " + testLog secondLineLog := "[May 01, 2017 19:00:04] " + testLog args := map[string]string{ - logDriverTypeKey: awslogsDriverName, - containerIDKey: testContainerID, - containerNameKey: testContainerName, + LogDriverTypeKey: AwslogsDriverName, + ContainerIDKey: TestContainerID, + ContainerNameKey: TestContainerName, awslogsCredentialsEndpointKey: testAwslogsCredentialEndpoint, awslogsRegionKey: testAwslogsRegion, awslogsGroupKey: testAwslogsGroup, @@ -161,7 +163,7 @@ var testAwslogs = func() { awslogsDatetimeFormatKey: testAwslogsDatetimeFormat, } creator := cio.BinaryIO(*Binary, args) - err := sendTestLogByContainerd(creator, fmt.Sprintf("%s\n%s\n%s", firstLineLog, secondLineLog, "[May 01, 2017 19:00:05]")) + err := SendTestLogByContainerd(creator, fmt.Sprintf("%s\n%s\n%s", firstLineLog, secondLineLog, "[May 01, 2017 19:00:05]")) gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) err = validateTestLogsInAwslogs(cwClient, testAwslogsGroup, nonExistentAwslogsStream, []string{fmt.Sprintf("%s\n", firstLineLog), fmt.Sprintf("%s\n", secondLineLog)}) @@ -171,9 +173,9 @@ var testAwslogs = func() { "when createGroup is false and createStream is true", func() { testLog := testLogPrefix + uuid.New().String() args := map[string]string{ - logDriverTypeKey: awslogsDriverName, - containerIDKey: testContainerID, - containerNameKey: testContainerName, + LogDriverTypeKey: AwslogsDriverName, + ContainerIDKey: TestContainerID, + ContainerNameKey: TestContainerName, awslogsCredentialsEndpointKey: testAwslogsCredentialEndpoint, awslogsRegionKey: testAwslogsRegion, awslogsGroupKey: testAwslogsGroup, @@ -183,7 +185,7 @@ var testAwslogs = func() { awslogsCreateStreamKey: "true", } creator := cio.BinaryIO(*Binary, args) - err := sendTestLogByContainerd(creator, testLog) + err := SendTestLogByContainerd(creator, testLog) gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) err = validateTestLogsInAwslogs(cwClient, testAwslogsGroup, nonExistentAwslogsStream, []string{testLog}) gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) @@ -192,9 +194,9 @@ var testAwslogs = func() { "when createGroup is false", func() { testLog := testLogPrefix + uuid.New().String() args := map[string]string{ - logDriverTypeKey: awslogsDriverName, - containerIDKey: testContainerID, - containerNameKey: testContainerName, + LogDriverTypeKey: AwslogsDriverName, + ContainerIDKey: TestContainerID, + ContainerNameKey: TestContainerName, awslogsCredentialsEndpointKey: testAwslogsCredentialEndpoint, awslogsRegionKey: testAwslogsRegion, awslogsGroupKey: nonExistentAwslogsGroup, @@ -203,7 +205,7 @@ var testAwslogs = func() { awslogsCreateGroupKey: "false", } creator := cio.BinaryIO(*Binary, args) - err := sendTestLogByContainerd(creator, testLog) + err := SendTestLogByContainerd(creator, testLog) gomega.Expect(err).Should(gomega.HaveOccurred()) gomega.Expect(err.Error()).Should(gomega.Equal(containerdTaskExitNonZeroMessage)) }) @@ -211,9 +213,9 @@ var testAwslogs = func() { "when createGroup is false and createStream is false", func() { testLog := testLogPrefix + uuid.New().String() args := map[string]string{ - logDriverTypeKey: awslogsDriverName, - containerIDKey: testContainerID, - containerNameKey: testContainerName, + LogDriverTypeKey: AwslogsDriverName, + ContainerIDKey: TestContainerID, + ContainerNameKey: TestContainerName, awslogsCredentialsEndpointKey: testAwslogsCredentialEndpoint, awslogsRegionKey: testAwslogsRegion, awslogsGroupKey: testAwslogsGroup, @@ -223,7 +225,7 @@ var testAwslogs = func() { awslogsCreateStreamKey: "false", } creator := cio.BinaryIO(*Binary, args) - err := sendTestLogByContainerd(creator, testLog) + err := SendTestLogByContainerd(creator, testLog) gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) err = validateTestLogsInAwslogs(cwClient, testAwslogsGroup, nonExistentAwslogsStream, []string{testLog}) gomega.Expect(err).Should(gomega.HaveOccurred()) @@ -232,9 +234,9 @@ var testAwslogs = func() { "when createGroup is true and createStream is true", func() { testLog := testLogPrefix + uuid.New().String() args := map[string]string{ - logDriverTypeKey: awslogsDriverName, - containerIDKey: testContainerID, - containerNameKey: testContainerName, + LogDriverTypeKey: AwslogsDriverName, + ContainerIDKey: TestContainerID, + ContainerNameKey: TestContainerName, awslogsCredentialsEndpointKey: testAwslogsCredentialEndpoint, awslogsRegionKey: testAwslogsRegion, awslogsGroupKey: nonExistentAwslogsGroup, @@ -244,7 +246,7 @@ var testAwslogs = func() { awslogsCreateStreamKey: "true", } creator := cio.BinaryIO(*Binary, args) - err := sendTestLogByContainerd(creator, testLog) + err := SendTestLogByContainerd(creator, testLog) gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) err = validateTestLogsInAwslogs(cwClient, nonExistentAwslogsGroup, nonExistentAwslogsStream, []string{testLog}) gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) @@ -253,9 +255,9 @@ var testAwslogs = func() { "is false and createStream is false", func() { testLog := testLogPrefix + uuid.New().String() args := map[string]string{ - logDriverTypeKey: awslogsDriverName, - containerIDKey: testContainerID, - containerNameKey: testContainerName, + LogDriverTypeKey: AwslogsDriverName, + ContainerIDKey: TestContainerID, + ContainerNameKey: TestContainerName, awslogsCredentialsEndpointKey: testAwslogsCredentialEndpoint, awslogsRegionKey: testAwslogsRegion, awslogsGroupKey: testAwslogsGroup, @@ -265,7 +267,7 @@ var testAwslogs = func() { awslogsCreateStreamKey: "false", } creator := cio.BinaryIO(*Binary, args) - err := sendTestLogByContainerd(creator, testLog) + err := SendTestLogByContainerd(creator, testLog) gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) err = validateTestLogsInAwslogs(cwClient, testAwslogsGroup, testAwslogsStream, []string{testLog}) gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) diff --git a/e2e/common.go b/e2e/common.go new file mode 100644 index 0000000..e67a018 --- /dev/null +++ b/e2e/common.go @@ -0,0 +1,93 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +//go:build e2e + +// Package e2e provides e2e tests of shim loggers for containerd. +package e2e + +import ( + "context" + "errors" + "fmt" + + "github.com/containerd/containerd" + "github.com/containerd/containerd/cio" + "github.com/containerd/containerd/namespaces" + "github.com/containerd/containerd/oci" +) + +const ( + // LogDriver options. + + // LogDriverTypeKey is the key of log driver type. + LogDriverTypeKey = "--log-driver" + // AwslogsDriverName is the name of awslogs driver. + AwslogsDriverName = "awslogs" + // FluentdDriverName is the name of fluentd driver. + FluentdDriverName = "fluentd" + // SplunkDriverName is the name of splunk driver. + SplunkDriverName = "splunk" + // ContainerIDKey is the key of the container id. + ContainerIDKey = "--container-id" + // ContainerNameKey is the key of the container name. + ContainerNameKey = "--container-name" + // TestContainerID is the id of the tes container. + TestContainerID = "210987654321" + // TestContainerName is the name of the test container. + TestContainerName = "test-container-name" + containerdAddress = "/run/containerd/containerd.sock" + testImage = "public.ecr.aws/docker/library/ubuntu:latest" + testLogPrefix = "test-e2e-log-" + containerdTaskExitNonZeroMessage = "\"containerd task exits with non-zero\"" +) + +// SendTestLogByContainerd sends a testLog to a specific shim logger by containerd. +func SendTestLogByContainerd(creator cio.Creator, testLog string) error { + // Create a new client connected to the containerd daemon + client, err := containerd.New(containerdAddress) + if err != nil { + return err + } + defer client.Close() //nolint:errcheck // closing client + // Create a new context with a customized namespace + ctx := namespaces.WithNamespace(context.Background(), "testShimLoggers") + // Pull an image + image, err := client.Pull(ctx, testImage, containerd.WithPullUnpack) + if err != nil { + return err + } // Create a new container with the pulled image + container, err := client.NewContainer(ctx, TestContainerID, containerd.WithImage(image), + containerd.WithNewSnapshot("test-snapshot", image), containerd.WithNewSpec(oci.WithImageConfig(image), + oci.WithProcessArgs("/bin/sh", "-c", fmt.Sprintf("printf \"%s\"", testLog)))) + if err != nil { + return err + } + defer container.Delete(ctx, containerd.WithSnapshotCleanup) //nolint:errcheck // testing only + // Create a new task from the container and start it + task, err := container.NewTask(ctx, creator) + if err != nil { + return err + } + defer task.Delete(ctx) //nolint:errcheck // testing only + + err = task.Start(ctx) + if err != nil { + return err + } + + statusC, err := task.Wait(ctx) + if err != nil { + return err + } + // Waiting for the task to finish + status := <-statusC + code, _, err := status.Result() + if err != nil { + return err + } + if code != uint32(0) { + return errors.New(containerdTaskExitNonZeroMessage) + } + return nil +} diff --git a/e2e/fluentd_test.go b/e2e/fluentd_test.go index 75ff54f..12e3200 100644 --- a/e2e/fluentd_test.go +++ b/e2e/fluentd_test.go @@ -1,6 +1,8 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 +//go:build e2e + package e2e import ( @@ -33,67 +35,67 @@ var testFluentd = func() { ginkgo.It("should send logs to fluentd log driver with default configs", func() { testLog := testLogPrefix + uuid.New().String() args := map[string]string{ - logDriverTypeKey: fluentdDriverName, - containerIDKey: testContainerID, - containerNameKey: testContainerName, + LogDriverTypeKey: FluentdDriverName, + ContainerIDKey: TestContainerID, + ContainerNameKey: TestContainerName, } creator := cio.BinaryIO(*Binary, args) - err := sendTestLogByContainerd(creator, testLog) + err := SendTestLogByContainerd(creator, testLog) gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) - validateTestLogsInFluentd(fluentdLogDirName, testLog, testContainerID) + validateTestLogsInFluentd(fluentdLogDirName, testLog, TestContainerID) }) ginkgo.It("should send logs to fluentd log driver with configs except tag being filled as default", func() { testLog := testLogPrefix + uuid.New().String() args := map[string]string{ - logDriverTypeKey: fluentdDriverName, - containerIDKey: testContainerID, - containerNameKey: testContainerName, + LogDriverTypeKey: FluentdDriverName, + ContainerIDKey: TestContainerID, + ContainerNameKey: TestContainerName, fluentdAddressKey: "localhost:24224", fluentdAsyncConnectKey: "false", fluentdSubSecondPrecisionKey: "true", fluentdBufferLimitKey: "1048576", } creator := cio.BinaryIO(*Binary, args) - err := sendTestLogByContainerd(creator, testLog) + err := SendTestLogByContainerd(creator, testLog) gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) - validateTestLogsInFluentd(fluentdLogDirName, testLog, testContainerID) + validateTestLogsInFluentd(fluentdLogDirName, testLog, TestContainerID) }) ginkgo.It("should send logs to fluentd log driver with async-connect is true", func() { testLog := testLogPrefix + uuid.New().String() args := map[string]string{ - logDriverTypeKey: fluentdDriverName, - containerIDKey: testContainerID, - containerNameKey: testContainerName, + LogDriverTypeKey: FluentdDriverName, + ContainerIDKey: TestContainerID, + ContainerNameKey: TestContainerName, fluentdAsyncConnectKey: "true", } creator := cio.BinaryIO(*Binary, args) - err := sendTestLogByContainerd(creator, testLog) + err := SendTestLogByContainerd(creator, testLog) gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) - validateTestLogsInFluentd(fluentdLogDirName, testLog, testContainerID) + validateTestLogsInFluentd(fluentdLogDirName, testLog, TestContainerID) }) ginkgo.It("should send logs to fluentd log driver with sub-second-precision is false", func() { testLog := testLogPrefix + uuid.New().String() args := map[string]string{ - logDriverTypeKey: fluentdDriverName, - containerIDKey: testContainerID, - containerNameKey: testContainerName, + LogDriverTypeKey: FluentdDriverName, + ContainerIDKey: TestContainerID, + ContainerNameKey: TestContainerName, fluentdSubSecondPrecisionKey: "false", } creator := cio.BinaryIO(*Binary, args) - err := sendTestLogByContainerd(creator, testLog) + err := SendTestLogByContainerd(creator, testLog) gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) - validateTestLogsInFluentd(fluentdLogDirName, testLog, testContainerID) + validateTestLogsInFluentd(fluentdLogDirName, testLog, TestContainerID) }) ginkgo.It("should send logs to fluentd log driver with customized tag", func() { testLog := testLogPrefix + uuid.New().String() args := map[string]string{ - logDriverTypeKey: fluentdDriverName, - containerIDKey: testContainerID, - containerNameKey: testContainerName, + LogDriverTypeKey: FluentdDriverName, + ContainerIDKey: TestContainerID, + ContainerNameKey: TestContainerName, fluentdTagKey: testFluentdTag, } creator := cio.BinaryIO(*Binary, args) - err := sendTestLogByContainerd(creator, testLog) + err := SendTestLogByContainerd(creator, testLog) gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) validateTestLogsInFluentd(fluentdLogDirName, testLog, testFluentdTag) }) diff --git a/e2e/main_test.go b/e2e/main_test.go index 7b18597..9a7dfee 100644 --- a/e2e/main_test.go +++ b/e2e/main_test.go @@ -1,39 +1,18 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 +//go:build e2e + package e2e import ( - "context" - "errors" "flag" - "fmt" "testing" - "github.com/containerd/containerd" - "github.com/containerd/containerd/cio" - "github.com/containerd/containerd/namespaces" - "github.com/containerd/containerd/oci" "github.com/onsi/ginkgo/v2" "github.com/onsi/gomega" ) -const ( - // LogDriver options. - logDriverTypeKey = "--log-driver" - awslogsDriverName = "awslogs" - fluentdDriverName = "fluentd" - splunkDriverName = "splunk" - containerIDKey = "--container-id" - containerNameKey = "--container-name" - testContainerID = "210987654321" - testContainerName = "test-container-name" - containerdAddress = "/run/containerd/containerd.sock" - testImage = "public.ecr.aws/docker/library/ubuntu:latest" - testLogPrefix = "test-e2e-log-" - containerdTaskExitNonZeroMessage = "\"containerd task exits with non-zero\"" -) - var ( // Binary is the path the binary of the shim loggers for containerd. Binary = flag.String("binary", "", "the binary of shim loggers for containerd") @@ -47,13 +26,13 @@ func TestShimLoggers(t *testing.T) { const description = "Shim loggers for containerd E2E Tests" ginkgo.Describe("", func() { - if *LogDriver == awslogsDriverName || *LogDriver == "" { + if *LogDriver == AwslogsDriverName || *LogDriver == "" { testAwslogs() } - if *LogDriver == fluentdDriverName || *LogDriver == "" { + if *LogDriver == FluentdDriverName || *LogDriver == "" { testFluentd() } - if *LogDriver == splunkDriverName || *LogDriver == "" { + if *LogDriver == SplunkDriverName || *LogDriver == "" { testSplunk(*SplunkToken) } }) @@ -61,52 +40,3 @@ func TestShimLoggers(t *testing.T) { gomega.RegisterFailHandler(ginkgo.Fail) ginkgo.RunSpecs(t, description) } - -func sendTestLogByContainerd(creator cio.Creator, testLog string) error { - // Create a new client connected to the containerd daemon - client, err := containerd.New(containerdAddress) - if err != nil { - return err - } - defer client.Close() //nolint:errcheck // closing client - // Create a new context with a customized namespace - ctx := namespaces.WithNamespace(context.Background(), "testShimLoggers") - // Pull an image - image, err := client.Pull(ctx, testImage, containerd.WithPullUnpack) - if err != nil { - return err - } // Create a new container with the pulled image - container, err := client.NewContainer(ctx, testContainerID, containerd.WithImage(image), - containerd.WithNewSnapshot("test-snapshot", image), containerd.WithNewSpec(oci.WithImageConfig(image), - oci.WithProcessArgs("/bin/sh", "-c", fmt.Sprintf("printf \"%s\"", testLog)))) - if err != nil { - return err - } - defer container.Delete(ctx, containerd.WithSnapshotCleanup) //nolint:errcheck // testing only - // Create a new task from the container and start it - task, err := container.NewTask(ctx, creator) - if err != nil { - return err - } - defer task.Delete(ctx) //nolint:errcheck // testing only - - err = task.Start(ctx) - if err != nil { - return err - } - - statusC, err := task.Wait(ctx) - if err != nil { - return err - } - // Waiting for the task to finish - status := <-statusC - code, _, err := status.Result() - if err != nil { - return err - } - if code != uint32(0) { - return errors.New(containerdTaskExitNonZeroMessage) - } - return nil -} diff --git a/e2e/splunk_test.go b/e2e/splunk_test.go index 2a2e8c5..3b23a62 100644 --- a/e2e/splunk_test.go +++ b/e2e/splunk_test.go @@ -1,6 +1,8 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 +//go:build e2e + package e2e import ( @@ -24,15 +26,15 @@ var testSplunk = func(token string) { ginkgo.It("should send logs to splunk log driver", func() { testLog := testLogPrefix + uuid.New().String() args := map[string]string{ - logDriverTypeKey: splunkDriverName, - containerIDKey: testContainerID, - containerNameKey: testContainerName, + LogDriverTypeKey: SplunkDriverName, + ContainerIDKey: TestContainerID, + ContainerNameKey: TestContainerName, splunkTokenKey: token, splunkURLkey: testSplunkURL, splunkInsecureskipverifyKey: "true", } creator := cio.BinaryIO(*Binary, args) - err := sendTestLogByContainerd(creator, testLog) + err := SendTestLogByContainerd(creator, testLog) gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) // TODO: Validate logs in Splunk local. https://github.com/aws/shim-loggers-for-containerd/issues/74 }) diff --git a/scripts/start-fluentd-local-endpoint b/scripts/start-fluentd-local-endpoint new file mode 100755 index 0000000..826a655 --- /dev/null +++ b/scripts/start-fluentd-local-endpoint @@ -0,0 +1,19 @@ +#!/bin/bash + +: # not using github action env because env map cannot be defined in terms of other vars in the map. public.ecr.aws/docker/library/fluentd:v1.16-debian-1 +FLUENTD_LOG_DIR=${GITHUB_WORKSPACE}/fluentd-logs +FLUENTD_PORT=24224 +FLUENTD_IMAGE=public.ecr.aws/docker/library/fluentd:v1.16-debian-1 +: # ECR Public anonymous pull rate quota is 1 per sec. https://docs.aws.amazon.com/AmazonECR/latest/public/public-service-quotas.html. +: # Running multiple jobs at the same time may exceed rate limit. Retry 3 times to prevent flaky behavior. +count=0 +until [ $count -ge 3 ] +do +docker pull $FLUENTD_IMAGE && break +count=$((count+1)) +[ $count -eq 3 ] && echo "Pull command failed after 3 attempts" && exit 1 +sleep 3 +done +: # Fluentd container is not using root user so need 777 to make it writable. https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#docker-container-filesystem +sudo mkdir -m 777 $FLUENTD_LOG_DIR +docker run -d -p $FLUENTD_PORT:24224 -p $FLUENTD_PORT:24224/udp -v $FLUENTD_LOG_DIR:/fluentd/log $FLUENTD_IMAGE diff --git a/scripts/start-localstack b/scripts/start-localstack new file mode 100755 index 0000000..368253c --- /dev/null +++ b/scripts/start-localstack @@ -0,0 +1,11 @@ +#!/bin/bash +# Copy from https://docs.localstack.cloud/user-guide/ci/github-actions/ + +LOCALSTACK_VERSION=2.2.0 +pip install localstack==${LOCALSTACK_VERSION} awscli-local[ver1] # install LocalStack cli and awslocal +docker pull localstack/localstack:${LOCALSTACK_VERSION} # Make sure to pull the latest version of the image +localstack start -d # Start LocalStack in the background + +echo "Waiting for LocalStack startup..." # Wait 30 seconds for the LocalStack container +localstack wait -t 30 # to become ready before timing out +echo "Startup complete"