Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: add MaxRetires config to cos, gcs and obs #147

Merged
merged 11 commits into from
Nov 11, 2024
Prev Previous commit
Next Next commit
add max_retries to gcs, obs, cos
Signed-off-by: Ashwanth Goli <iamashwanth@gmail.com>
  • Loading branch information
ashwanthgoli committed Oct 23, 2024
commit c7d91edfa809f55afc2dd96eb4e791c220677249
5 changes: 5 additions & 0 deletions providers/cos/cos.go
Original file line number Diff line number Diff line change
@@ -59,6 +59,7 @@ type Config struct {
Endpoint string `yaml:"endpoint"`
SecretKey string `yaml:"secret_key"`
SecretId string `yaml:"secret_id"`
MaxRetries int `yaml:"max_retries"`
HTTPConfig exthttp.HTTPConfig `yaml:"http_config"`
}

@@ -143,6 +144,10 @@ func NewBucketWithConfig(logger log.Logger, config Config, component string, rt
},
})

if config.MaxRetries > 0 {
client.Conf.RetryOpt.Count = config.MaxRetries
}

bkt := &Bucket{
logger: logger,
client: client,
10 changes: 6 additions & 4 deletions providers/gcs/gcs.go
Original file line number Diff line number Diff line change
@@ -55,8 +55,10 @@ type Config struct {
ChunkSizeBytes int `yaml:"chunk_size_bytes"`
noAuth bool `yaml:"no_auth"`

// gcs client retries idempotent operations by default, this option disables retries.
DisableRetries bool `yaml:"disable_retries"`
// MaxRetries controls the number of retries for idempotent operations.
// Overrides the default gcs storage client behaviour if this value is greater than 0.
// Set this to 1 to disable retries.
MaxRetries int `yaml:"max_retries"`
}

// Bucket implements the store.Bucket and shipper.Bucket interfaces against GCS.
@@ -176,8 +178,8 @@ func newBucket(ctx context.Context, logger log.Logger, gc Config, opts []option.
chunkSize: gc.ChunkSizeBytes,
}

if gc.DisableRetries {
bkt.bkt = bkt.bkt.Retryer(storage.WithPolicy(storage.RetryNever))
if gc.MaxRetries > 0 {
bkt.bkt = bkt.bkt.Retryer(storage.WithMaxAttempts(gc.MaxRetries))
}

return bkt, nil
9 changes: 8 additions & 1 deletion providers/obs/obs.go
Original file line number Diff line number Diff line change
@@ -46,6 +46,7 @@ type Config struct {
Endpoint string `yaml:"endpoint"`
AccessKey string `yaml:"access_key"`
SecretKey string `yaml:"secret_key"`
MaxRetries int `yaml:"max_retries"`
HTTPConfig exthttp.HTTPConfig `yaml:"http_config"`
}

@@ -102,7 +103,13 @@ func NewBucketWithConfig(logger log.Logger, config Config) (*Bucket, error) {
return nil, errors.Wrap(err, "get http transport err")
}

client, err := obs.New(config.AccessKey, config.SecretKey, config.Endpoint, obs.WithHttpTransport(rt))
var client *obs.ObsClient
if config.MaxRetries > 0 {
client, err = obs.New(config.AccessKey, config.SecretKey, config.Endpoint, obs.WithHttpTransport(rt), obs.WithMaxRetryCount(config.MaxRetries))
} else {
client, err = obs.New(config.AccessKey, config.SecretKey, config.Endpoint, obs.WithHttpTransport(rt))
}

if err != nil {
return nil, errors.Wrap(err, "initialize obs client err")
}