From c8184e2540221cc92ec93c4be7a4f8b9c755286c Mon Sep 17 00:00:00 2001 From: Derek Su Date: Tue, 2 Jul 2024 16:12:32 +0000 Subject: [PATCH 1/5] chore(dockerile): use registry.suse.com/bci/golang:1.22 base image Longhorn 8880 Signed-off-by: Derek Su --- Dockerfile.dapper | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/Dockerfile.dapper b/Dockerfile.dapper index 6d430ed..fb9fbd8 100644 --- a/Dockerfile.dapper +++ b/Dockerfile.dapper @@ -1,25 +1,19 @@ -FROM ubuntu:16.04 -# FROM arm=armhf/ubuntu:16.04 +FROM registry.suse.com/bci/golang:1.22 ARG DAPPER_HOST_ARCH=amd64 ARG http_proxy ARG https_proxy ENV HOST_ARCH=${DAPPER_HOST_ARCH} ARCH=${DAPPER_HOST_ARCH} -RUN apt-get update && \ - apt-get install -y gcc ca-certificates git wget curl vim less file && \ - rm -f /bin/sh && ln -s /bin/bash /bin/sh +RUN zypper -n addrepo --refresh https://download.opensuse.org/repositories/Base:System/openSUSE_Factory/Base:System.repo && \ + zypper --gpg-auto-import-keys ref -ENV DOCKER_URL_amd64=https://get.docker.com/builds/Linux/x86_64/docker-1.10.3 \ - DOCKER_URL_arm64=https://github.com/rancher/docker/releases/download/v1.10.3-ros1/docker-1.10.3_arm \ - DOCKER_URL=DOCKER_URL_${ARCH} - -RUN wget -O - ${!DOCKER_URL} > /usr/bin/docker && chmod +x /usr/bin/docker +RUN zypper -n install glibc glibc-static gcc ca-certificates git wget curl vim less file awk docker && \ + rm -rf /var/cache/zypp/* ENV GOLANG_ARCH_amd64=amd64 GOLANG_ARCH_arm64=arm64 GOLANG_ARCH=GOLANG_ARCH_${ARCH} \ GOPATH=/go PATH=/go/bin:/usr/local/go/bin:${PATH} SHELL=/bin/bash -RUN wget -O - https://storage.googleapis.com/golang/go1.22.2.linux-${!GOLANG_ARCH}.tar.gz | tar -xzf - -C /usr/local RUN curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.55.2 ENV DAPPER_SOURCE /go/src/github.com/longhorn/sparse-tools From 52cff0a6f518eaeb1a678e3576558813a21b2562 Mon Sep 17 00:00:00 2001 From: Derek Su Date: Tue, 2 Jul 2024 16:11:18 +0000 Subject: [PATCH 2/5] chore(test): enable verbose output Longhorn 8880 Signed-off-by: Derek Su --- scripts/test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/test b/scripts/test index a1785bd..4f24f62 100755 --- a/scripts/test +++ b/scripts/test @@ -7,4 +7,4 @@ echo Running tests PACKAGES="$(find -name '*.go' | xargs -I{} dirname {} | cut -f2 -d/ | sort -u | grep -Ev '(^\.$|.git|.trash-cache|vendor|bin)' | sed -e 's!^!./!' -e 's!$!/...!')" -go test -race -cover -tags=test ${PACKAGES} -coverprofile=coverage.out +go test -v -race -cover -tags=test ${PACKAGES} -coverprofile=coverage.out From 1dec90143e27ce63a5a45e7ade4b09feba4823c8 Mon Sep 17 00:00:00 2001 From: Derek Su Date: Tue, 2 Jul 2024 16:08:40 +0000 Subject: [PATCH 3/5] fix(test): fix failed test cases due to race condition Longhorn 8880 Signed-off-by: Derek Su --- sparse/test/client_test.go | 74 +++++++++++++++++++++++++++++++++----- sparse/test/fiemap_test.go | 8 ++++- sparse/test/ssync_test.go | 17 +++++++-- 3 files changed, 88 insertions(+), 11 deletions(-) diff --git a/sparse/test/client_test.go b/sparse/test/client_test.go index 2575b72..cbb6eea 100644 --- a/sparse/test/client_test.go +++ b/sparse/test/client_test.go @@ -4,6 +4,7 @@ import ( "context" "crypto/rand" "os" + "sync" "testing" "github.com/stretchr/testify/assert" @@ -251,11 +252,20 @@ func TestSyncAnyFile(t *testing.T) { func testSyncAnyFile(t *testing.T, src, dst string, directIO, fastSync bool) { // Sync + var wg sync.WaitGroup + wg.Add(1) go func() { - _ = rest.TestServer(context.Background(), port, dst, timeout) + defer wg.Done() + + err := rest.TestServer(context.Background(), port, dst, timeout) + // http server is closed by the client after file transfer is done, so the error + // "http: Server closed" is expected. + assert.True(t, err == nil || err.Error() == "http: Server closed", "Unexpected error: %v", err) }() err := SyncFile(src, localhost+":"+port, timeout, directIO, fastSync) + wg.Wait() + // Verify if err != nil { t.Fatal("sync error") @@ -271,12 +281,20 @@ func testSyncAnyFile(t *testing.T, src, dst string, directIO, fastSync bool) { func testSyncAnyFileExpectFailure(t *testing.T, src, dst string, directIO, fastSync bool) { // Sync + var wg sync.WaitGroup + wg.Add(1) go func() { + defer wg.Done() + err := rest.TestServer(context.Background(), port, dst, timeout) - assert.Nil(t, err) + // http server is closed by the client after file transfer is done, so the error + // "http: Server closed" is expected. + assert.True(t, err == nil || err.Error() == "http: Server closed", "Unexpected error: %v", err) }() err := SyncFile(src, localhost+":"+port, timeout, directIO, fastSync) + wg.Wait() + // Verify if err == nil { t.Fatal("sync error") @@ -733,12 +751,20 @@ func testSyncFile(t *testing.T, layoutLocal, layoutRemote []FileInterval, direct } // Sync + var wg sync.WaitGroup + wg.Add(1) go func() { + defer wg.Done() + err := rest.TestServer(context.Background(), port, remotePath, timeout) - assert.Nil(t, err) + // http server is closed by the client after file transfer is done, so the error + // "http: Server closed" is expected. + assert.True(t, err == nil || err.Error() == "http: Server closed", "Unexpected error: %v", err) }() err := SyncFile(localPath, localhost+":"+port, timeout, true /* directIO */, false /* fastSync */) + wg.Wait() + // Verify if err != nil { t.Fatal("sync error") @@ -775,37 +801,61 @@ func Benchmark_1G_InitFiles(b *testing.B) { } func Benchmark_1G_SendFiles_Whole(b *testing.B) { + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + err := rest.TestServer(context.Background(), port, remoteBigPath, timeout) - assert.Nil(b, err) + // http server is closed by the client after file transfer is done, so the error + // "http: Server closed" is expected. + assert.True(b, err == nil || err.Error() == "http: Server closed", "Unexpected error: %v", err) }() err := SyncFile(localBigPath, localhost+":"+port, timeout, true /* directIO */, false /* fastSync */) + wg.Wait() + if err != nil { b.Fatal("sync error") } } func Benchmark_1G_SendFiles_Whole_No_DirectIO(b *testing.B) { + var wg sync.WaitGroup + wg.Add(1) go func() { + defer wg.Done() + err := rest.TestServer(context.Background(), port, remoteBigPath, timeout) - assert.Nil(b, err) + // http server is closed by the client after file transfer is done, so the error + // "http: Server closed" is expected. + assert.True(b, err == nil || err.Error() == "http: Server closed", "Unexpected error: %v", err) }() err := SyncFile(localBigPath, localhost+":"+port, timeout, false /* directIO */, false /* fastSync */) + wg.Wait() + if err != nil { b.Fatal("sync error") } } func Benchmark_1G_SendFiles_Diff(b *testing.B) { - + var wg sync.WaitGroup + wg.Add(1) go func() { + defer wg.Done() + err := rest.TestServer(context.Background(), port, remoteBigPath, timeout) - assert.Nil(b, err) + // http server is closed by the client after file transfer is done, so the error + // "http: Server closed" is expected. + assert.True(b, err == nil || err.Error() == "http: Server closed", "Unexpected error: %v", err) }() err := SyncFile(localBigPath, localhost+":"+port, timeout, true /* directIO */, false /* fastSync */) + wg.Wait() + if err != nil { b.Fatal("sync error") } @@ -867,12 +917,20 @@ func TestSyncSnapshotZeroByte(t *testing.T) { func testSyncAnyContent(t *testing.T, snapshotName string, dstFileName string, rw ReaderWriterAt, snapshotSize int64) { // Sync + var wg sync.WaitGroup + wg.Add(1) go func() { + defer wg.Done() + err := rest.TestServer(context.Background(), port, dstFileName, timeout) - assert.Nil(t, err) + // http server is closed by the client after file transfer is done, so the error + // "http: Server closed" is expected. + assert.True(t, err == nil || err.Error() == "http: Server closed", "Unexpected error: %v", err) }() err := SyncContent(snapshotName, rw, snapshotSize, localhost+":"+port, timeout, true, false) + wg.Wait() + // Verify if err != nil { t.Fatalf("sync error: %v", err) diff --git a/sparse/test/fiemap_test.go b/sparse/test/fiemap_test.go index ddb47ea..5660b7a 100644 --- a/sparse/test/fiemap_test.go +++ b/sparse/test/fiemap_test.go @@ -6,6 +6,7 @@ import ( "math/rand" "os" "path/filepath" + "sync" "testing" "time" @@ -54,9 +55,13 @@ func TestFileSync(t *testing.T) { // defer fileCleanup(dstPath) log.Info("Syncing file...") startTime := time.Now() + var wg sync.WaitGroup + wg.Add(1) go func() { + defer wg.Done() + err := rest.TestServer(context.Background(), port, dstPath, timeout) - assert.Nil(t, err) + assert.True(t, err == nil || err.Error() == "http: Server closed", "Unexpected error: %v", err) }() time.Sleep(time.Second) err := SyncFile(srcPath, localhost+":"+port, timeout, true, false) @@ -64,6 +69,7 @@ func TestFileSync(t *testing.T) { t.Fatalf("sync error: %v", err) } log.Infof("Syncing done, size: %v elapsed: %.2fs", testFileSize, time.Since(startTime).Seconds()) + wg.Wait() startTime = time.Now() log.Info("Checking...") diff --git a/sparse/test/ssync_test.go b/sparse/test/ssync_test.go index c050b24..6784587 100644 --- a/sparse/test/ssync_test.go +++ b/sparse/test/ssync_test.go @@ -8,6 +8,7 @@ import ( "os" "strconv" "strings" + "sync" "testing" "time" @@ -198,10 +199,15 @@ func RandomSync(t *testing.T, size, seed int64, srcPath, dstPath string, dstCrea } log.Infof("Syncing with directIO: %v size: %v", directIO, size) + var wg sync.WaitGroup + wg.Add(1) go func() { + defer wg.Done() + err := rest.TestServer(context.Background(), port, dstPath, timeout) - assert.Nil(t, err) + assert.True(t, err == nil || err.Error() == "http: Server closed", "Unexpected error: %v", err) }() + startTime := time.Now() err := SyncFile(srcPath, localhost+":"+port, timeout, directIO, fastSync) if err != nil { @@ -209,6 +215,8 @@ func RandomSync(t *testing.T, size, seed int64, srcPath, dstPath string, dstCrea } log.Infof("Syncing done, size: %v elapsed: %.2fs", size, time.Since(startTime).Seconds()) + wg.Wait() + startTime = time.Now() err = checkSparseFiles(srcPath, dstPath) if err != nil { @@ -264,9 +272,13 @@ func TestSyncCancellation(t *testing.T) { dstName := tempFilePath(dstPrefix) ctx, cancelFunc := context.WithCancel(context.Background()) + var wg sync.WaitGroup + wg.Add(1) go func() { + defer wg.Done() + err := rest.TestServer(ctx, port, dstName, timeout) - assert.Nil(t, err) + assert.True(t, err == nil || err.Error() == "http: Server closed", "Unexpected error: %v", err) }() client := http.Client{} @@ -311,4 +323,5 @@ func TestSyncCancellation(t *testing.T) { if httpErr == nil || !strings.Contains(httpErr.Error(), "connection refused") { t.Fatalf("Unexpected error: %v", httpErr) } + wg.Wait() } From 853c7387822f5ef064c5892e5ccaf0ae21aac6e2 Mon Sep 17 00:00:00 2001 From: Derek Su Date: Tue, 2 Jul 2024 16:53:02 +0000 Subject: [PATCH 4/5] chore(go): update go.mod Longhorn 8880 Signed-off-by: Derek Su --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index a2be39b..9ffe4d5 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/longhorn/sparse-tools -go 1.22 +go 1.22.0 require ( github.com/google/uuid v1.6.0 From bd6465308bc86e3c4da379dd5b764a50391a8789 Mon Sep 17 00:00:00 2001 From: Derek Su Date: Wed, 3 Jul 2024 00:50:54 +0000 Subject: [PATCH 5/5] chore(ci): remove .drone.yml Longhorn 8880 Signed-off-by: Derek Su --- .drone.yml | 6 ------ 1 file changed, 6 deletions(-) delete mode 100644 .drone.yml diff --git a/.drone.yml b/.drone.yml deleted file mode 100644 index 9a8850f..0000000 --- a/.drone.yml +++ /dev/null @@ -1,6 +0,0 @@ -build: - image: rancher/dapper:1.10.3 - volumes: - - /var/run/docker.sock:/var/run/docker.sock - commands: - - dapper ci