fix(test): fix failed test cases due to race condition

Longhorn 8880

Signed-off-by: Derek Su <[email protected]>

sparse/test/client_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"crypto/rand"
 	"os"
+	"sync"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -251,11 +252,20 @@ func TestSyncAnyFile(t *testing.T) {
 
 func testSyncAnyFile(t *testing.T, src, dst string, directIO, fastSync bool) {
 	// Sync
+	var wg sync.WaitGroup
+	wg.Add(1)
 	go func() {
-		_ = rest.TestServer(context.Background(), port, dst, timeout)
+		defer wg.Done()
+
+		err := rest.TestServer(context.Background(), port, dst, timeout)
+		// http server is closed by the client after file transfer is done, so the error
+		// "http: Server closed" is expected.
+		assert.True(t, err == nil || err.Error() == "http: Server closed", "Unexpected error: %v", err)
 	}()
 	err := SyncFile(src, localhost+":"+port, timeout, directIO, fastSync)
 
+	wg.Wait()
+
 	// Verify
 	if err != nil {
 		t.Fatal("sync error")
@@ -271,12 +281,20 @@ func testSyncAnyFile(t *testing.T, src, dst string, directIO, fastSync bool) {
 
 func testSyncAnyFileExpectFailure(t *testing.T, src, dst string, directIO, fastSync bool) {
 	// Sync
+	var wg sync.WaitGroup
+	wg.Add(1)
 	go func() {
+		defer wg.Done()
+
 		err := rest.TestServer(context.Background(), port, dst, timeout)
-		assert.Nil(t, err)
+		// http server is closed by the client after file transfer is done, so the error
+		// "http: Server closed" is expected.
+		assert.True(t, err == nil || err.Error() == "http: Server closed", "Unexpected error: %v", err)
 	}()
 	err := SyncFile(src, localhost+":"+port, timeout, directIO, fastSync)
 
+	wg.Wait()
+
 	// Verify
 	if err == nil {
 		t.Fatal("sync error")
@@ -733,12 +751,20 @@ func testSyncFile(t *testing.T, layoutLocal, layoutRemote []FileInterval, direct
 	}
 
 	// Sync
+	var wg sync.WaitGroup
+	wg.Add(1)
 	go func() {
+		defer wg.Done()
+
 		err := rest.TestServer(context.Background(), port, remotePath, timeout)
-		assert.Nil(t, err)
+		// http server is closed by the client after file transfer is done, so the error
+		// "http: Server closed" is expected.
+		assert.True(t, err == nil || err.Error() == "http: Server closed", "Unexpected error: %v", err)
 	}()
 	err := SyncFile(localPath, localhost+":"+port, timeout, true /* directIO */, false /* fastSync */)
 
+	wg.Wait()
+
 	// Verify
 	if err != nil {
 		t.Fatal("sync error")
@@ -775,37 +801,61 @@ func Benchmark_1G_InitFiles(b *testing.B) {
 }
 
 func Benchmark_1G_SendFiles_Whole(b *testing.B) {
+	var wg sync.WaitGroup
+	wg.Add(1)
+
 	go func() {
+		defer wg.Done()
+
 		err := rest.TestServer(context.Background(), port, remoteBigPath, timeout)
-		assert.Nil(b, err)
+		// http server is closed by the client after file transfer is done, so the error
+		// "http: Server closed" is expected.
+		assert.True(b, err == nil || err.Error() == "http: Server closed", "Unexpected error: %v", err)
 	}()
 	err := SyncFile(localBigPath, localhost+":"+port, timeout, true /* directIO */, false /* fastSync */)
 
+	wg.Wait()
+
 	if err != nil {
 		b.Fatal("sync error")
 	}
 }
 
 func Benchmark_1G_SendFiles_Whole_No_DirectIO(b *testing.B) {
+	var wg sync.WaitGroup
+	wg.Add(1)
 	go func() {
+		defer wg.Done()
+
 		err := rest.TestServer(context.Background(), port, remoteBigPath, timeout)
-		assert.Nil(b, err)
+		// http server is closed by the client after file transfer is done, so the error
+		// "http: Server closed" is expected.
+		assert.True(b, err == nil || err.Error() == "http: Server closed", "Unexpected error: %v", err)
 	}()
 	err := SyncFile(localBigPath, localhost+":"+port, timeout, false /* directIO */, false /* fastSync */)
 
+	wg.Wait()
+
 	if err != nil {
 		b.Fatal("sync error")
 	}
 }
 
 func Benchmark_1G_SendFiles_Diff(b *testing.B) {
-
+	var wg sync.WaitGroup
+	wg.Add(1)
 	go func() {
+		defer wg.Done()
+
 		err := rest.TestServer(context.Background(), port, remoteBigPath, timeout)
-		assert.Nil(b, err)
+		// http server is closed by the client after file transfer is done, so the error
+		// "http: Server closed" is expected.
+		assert.True(b, err == nil || err.Error() == "http: Server closed", "Unexpected error: %v", err)
 	}()
 	err := SyncFile(localBigPath, localhost+":"+port, timeout, true /* directIO */, false /* fastSync */)
 
+	wg.Wait()
+
 	if err != nil {
 		b.Fatal("sync error")
 	}
@@ -867,12 +917,20 @@ func TestSyncSnapshotZeroByte(t *testing.T) {
 
 func testSyncAnyContent(t *testing.T, snapshotName string, dstFileName string, rw ReaderWriterAt, snapshotSize int64) {
 	// Sync
+	var wg sync.WaitGroup
+	wg.Add(1)
 	go func() {
+		defer wg.Done()
+
 		err := rest.TestServer(context.Background(), port, dstFileName, timeout)
-		assert.Nil(t, err)
+		// http server is closed by the client after file transfer is done, so the error
+		// "http: Server closed" is expected.
+		assert.True(t, err == nil || err.Error() == "http: Server closed", "Unexpected error: %v", err)
 	}()
 	err := SyncContent(snapshotName, rw, snapshotSize, localhost+":"+port, timeout, true, false)
 
+	wg.Wait()
+
 	// Verify
 	if err != nil {
 		t.Fatalf("sync error: %v", err)

sparse/test/fiemap_test.go

@@ -6,6 +6,7 @@ import (
 	"math/rand"
 	"os"
 	"path/filepath"
+	"sync"
 	"testing"
 	"time"
 
@@ -54,16 +55,21 @@ func TestFileSync(t *testing.T) {
 	// defer fileCleanup(dstPath)
 	log.Info("Syncing file...")
 	startTime := time.Now()
+	var wg sync.WaitGroup
+	wg.Add(1)
 	go func() {
+		defer wg.Done()
+
 		err := rest.TestServer(context.Background(), port, dstPath, timeout)
-		assert.Nil(t, err)
+		assert.True(t, err == nil || err.Error() == "http: Server closed", "Unexpected error: %v", err)
 	}()
 	time.Sleep(time.Second)
 	err := SyncFile(srcPath, localhost+":"+port, timeout, true, false)
 	if err != nil {
 		t.Fatalf("sync error: %v", err)
 	}
 	log.Infof("Syncing done, size: %v elapsed: %.2fs", testFileSize, time.Since(startTime).Seconds())
+	wg.Wait()
 
 	startTime = time.Now()
 	log.Info("Checking...")

sparse/test/ssync_test.go

@@ -8,6 +8,7 @@ import (
 	"os"
 	"strconv"
 	"strings"
+	"sync"
 	"testing"
 	"time"
 
@@ -198,17 +199,24 @@ func RandomSync(t *testing.T, size, seed int64, srcPath, dstPath string, dstCrea
 	}
 
 	log.Infof("Syncing with directIO: %v size: %v", directIO, size)
+	var wg sync.WaitGroup
+	wg.Add(1)
 	go func() {
+		defer wg.Done()
+
 		err := rest.TestServer(context.Background(), port, dstPath, timeout)
-		assert.Nil(t, err)
+		assert.True(t, err == nil || err.Error() == "http: Server closed", "Unexpected error: %v", err)
 	}()
+
 	startTime := time.Now()
 	err := SyncFile(srcPath, localhost+":"+port, timeout, directIO, fastSync)
 	if err != nil {
 		t.Fatal("sync error")
 	}
 	log.Infof("Syncing done, size: %v elapsed: %.2fs", size, time.Since(startTime).Seconds())
 
+	wg.Wait()
+
 	startTime = time.Now()
 	err = checkSparseFiles(srcPath, dstPath)
 	if err != nil {
@@ -264,9 +272,13 @@ func TestSyncCancellation(t *testing.T) {
 	dstName := tempFilePath(dstPrefix)
 
 	ctx, cancelFunc := context.WithCancel(context.Background())
+	var wg sync.WaitGroup
+	wg.Add(1)
 	go func() {
+		defer wg.Done()
+
 		err := rest.TestServer(ctx, port, dstName, timeout)
-		assert.Nil(t, err)
+		assert.True(t, err == nil || err.Error() == "http: Server closed", "Unexpected error: %v", err)
 	}()
 
 	client := http.Client{}
@@ -311,4 +323,5 @@ func TestSyncCancellation(t *testing.T) {
 	if httpErr == nil || !strings.Contains(httpErr.Error(), "connection refused") {
 		t.Fatalf("Unexpected error: %v", httpErr)
 	}
+	wg.Wait()
 }