S3 bucket: Don't retry when ctx cancel (#5997)

* s3: no need to retry and wait when last error is context cancel or deadline exceeded Signed-off-by: Ben Ye <[email protected]> * add tests Signed-off-by: Ben Ye <[email protected]> * lint Signed-off-by: Ben Ye <[email protected]> --------- Signed-off-by: Ben Ye <[email protected]>
cortexproject · Jun 4, 2024 · 6fe41ac · 6fe41ac
1 parent 35732c3
commit 6fe41ac
Show file tree

Hide file tree

Showing 2 changed files with 13 additions and 0 deletions.
diff --git a/pkg/storage/bucket/s3/bucket_client.go b/pkg/storage/bucket/s3/bucket_client.go
@@ -8,6 +8,7 @@ import (
 
 	"github.com/go-kit/log"
 	"github.com/go-kit/log/level"
+	"github.com/pkg/errors"
 	"github.com/prometheus/common/model"
 	"github.com/thanos-io/objstore"
 	"github.com/thanos-io/objstore/providers/s3"
@@ -127,6 +128,10 @@ func (b *BucketWithRetries) retry(ctx context.Context, f func() error, operation
 		if lastErr == nil {
 			return nil
 		}
+		// No need to retry when context was already canceled.
+		if errors.Is(lastErr, context.Canceled) || errors.Is(lastErr, context.DeadlineExceeded) {
+			return lastErr
+		}
 		if b.bucket.IsObjNotFoundErr(lastErr) || b.bucket.IsAccessDeniedErr(lastErr) {
 			return lastErr
 		}

diff --git a/pkg/storage/bucket/s3/bucket_client_test.go b/pkg/storage/bucket/s3/bucket_client_test.go
@@ -34,6 +34,14 @@ func TestBucketWithRetries_ShouldRetry(t *testing.T) {
 			err:        errKeyDenied,
 			retryCount: 1,
 		},
+		"should not retry when context canceled": {
+			err:        context.Canceled,
+			retryCount: 1,
+		},
+		"should not retry when context deadline exceeded": {
+			err:        context.DeadlineExceeded,
+			retryCount: 1,
+		},
 	}
 
 	for name, tc := range cases {