From 21ec012e89eff48d94feae6508b3bfdfe2a253a7 Mon Sep 17 00:00:00 2001 From: yangsen Date: Fri, 26 Apr 2024 14:59:08 +0800 Subject: [PATCH] Get region support specified UC hosts and Client & List bucket support output file parts (#125) --- CHANGELOG.md | 5 ++++ README.md | 2 +- conf/conf.go | 2 +- storage/bucket.go | 2 +- storage/bucket_list.go | 17 ++++++++++++-- storage/bucket_list_test.go | 47 +++++++++++++++++++++++++++++++++++++ storage/region.go | 40 +++++++++++++++++++++---------- storage/region_test.go | 25 +++++++++++++++++++- storage/region_uc_v2.go | 34 +++++++++++++++++++++++---- storage/region_uc_v4.go | 8 +++++-- 10 files changed, 157 insertions(+), 25 deletions(-) create mode 100644 storage/bucket_list_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index facef05b..442a619d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,9 @@ # Changelog +## 7.20.1 +* 新增 + * 获取区域 API 支持单独配置 UC 域名 + * BucketManager List Bucket 接口支持返回文件的 parts + ## 7.20.0 * 新增 * 新版存储客户端库 storagev2 包，包含 diff --git a/README.md b/README.md index c6c289e0..8a565d8e 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ github.com/qiniu/go-sdk 在您的项目中的 `go.mod` 文件内添加这行代码 ``` -require github.com/qiniu/go-sdk/v7 v7.20.0 +require github.com/qiniu/go-sdk/v7 v7.20.1 ``` 并且在项目中使用 `"github.com/qiniu/go-sdk/v7"` 引用 Qiniu Go SDK。 diff --git a/conf/conf.go b/conf/conf.go index edb5395e..6c2536fd 100644 --- a/conf/conf.go +++ b/conf/conf.go @@ -5,7 +5,7 @@ import ( "strings" ) -const Version = "7.20.0" +const Version = "7.20.1" const ( CONTENT_TYPE_JSON = "application/json" diff --git a/storage/bucket.go b/storage/bucket.go index 9cb1b86d..83c6a584 100644 --- a/storage/bucket.go +++ b/storage/bucket.go @@ -1023,7 +1023,7 @@ func (m *BucketManager) Zone(bucket string) (z *Zone, err error) { } func (m *BucketManager) makeRequestOptions() *apis.Options { - return &apis.Options{OverwrittenBucketHosts: getUcEndpoint(m.Cfg.UseHTTPS)} + return &apis.Options{OverwrittenBucketHosts: getUcEndpoint(m.Cfg.UseHTTPS, nil)} } // 构建op的方法，导出的方法支持在Batch操作中使用 diff --git a/storage/bucket_list.go b/storage/bucket_list.go index 59f9fda8..0e081d31 100644 --- a/storage/bucket_list.go +++ b/storage/bucket_list.go @@ -54,6 +54,11 @@ type ListItem struct { * 文件的 md5 值 */ Md5 string `json:"md5"` + + /** + * 文件的分片信息 + */ + Parts []uint `json:"parts"` } // 接口可能返回空的记录 @@ -103,6 +108,7 @@ type listInputOptions struct { delimiter string marker string limit int + needParts bool } type ListInputOption func(options *listInputOptions) @@ -131,6 +137,12 @@ func ListInputOptionsLimit(limit int) ListInputOption { } } +func ListInputOptionsNeedParts(needParts bool) ListInputOption { + return func(input *listInputOptions) { + input.needParts = needParts + } +} + // ListFilesWithContext // // @Description: 用来获取空间文件列表，可以根据需要指定文件的列举条件 @@ -166,7 +178,7 @@ func (m *BucketManager) ListFilesWithContext(ctx context.Context, bucket string, } ret = &ListFilesRet{} - reqURL := fmt.Sprintf("%s%s", host, uriListFiles(bucket, inputOptions.prefix, inputOptions.delimiter, inputOptions.marker, inputOptions.limit)) + reqURL := fmt.Sprintf("%s%s", host, uriListFiles(bucket, inputOptions.prefix, inputOptions.delimiter, inputOptions.marker, inputOptions.limit, inputOptions.needParts)) err = m.Client.CredentialedCall(ctx, m.Mac, auth.TokenQiniu, ret, "POST", reqURL, nil) if err != nil { return nil, false, err @@ -228,7 +240,7 @@ func (m *BucketManager) ListBucketContext(ctx context.Context, bucket, prefix, d return retCh, err } -func uriListFiles(bucket, prefix, delimiter, marker string, limit int) string { +func uriListFiles(bucket, prefix, delimiter, marker string, limit int, needParts bool) string { query := make(url.Values) query.Add("bucket", bucket) if prefix != "" { @@ -243,5 +255,6 @@ func uriListFiles(bucket, prefix, delimiter, marker string, limit int) string { if limit > 0 { query.Add("limit", strconv.FormatInt(int64(limit), 10)) } + query.Add("needparts", strconv.FormatBool(needParts)) return fmt.Sprintf("/list?%s", query.Encode()) } diff --git a/storage/bucket_list_test.go b/storage/bucket_list_test.go new file mode 100644 index 00000000..add78055 --- /dev/null +++ b/storage/bucket_list_test.go @@ -0,0 +1,47 @@ +//go:build integration +// +build integration + +package storage + +import ( + "context" + "testing" +) + +func TestList(t *testing.T) { + ret, _, err := bucketManager.ListFilesWithContext(context.Background(), testBucket, + ListInputOptionsLimit(1000), + ListInputOptionsNeedParts(false), + ) + if err != nil { + t.Fatalf("List bucket files error: %v\n", err) + } + + hasParts := false + for _, item := range ret.Items { + if len(item.Parts) > 0 { + hasParts = true + } + } + if hasParts { + t.Fatal("list files: should no parts") + } + + ret, _, err = bucketManager.ListFilesWithContext(context.Background(), testBucket, + ListInputOptionsLimit(1000), + ListInputOptionsNeedParts(true), + ) + if err != nil { + t.Fatalf("List bucket files error: %v\n", err) + } + + hasParts = false + for _, item := range ret.Items { + if len(item.Parts) > 0 { + hasParts = true + } + } + if !hasParts { + t.Fatal("list files: should parts") + } +} diff --git a/storage/region.go b/storage/region.go index 6b34958b..5abdf9e4 100644 --- a/storage/region.go +++ b/storage/region.go @@ -290,18 +290,21 @@ func getUcBackupHosts() []string { return hosts } -func getUcEndpoint(useHttps bool) region_v2.EndpointsProvider { - ucHosts := make([]string, 0, 1+len(ucHosts)) - if len(UcHost) > 0 { - ucHosts = append(ucHosts, endpoint(useHttps, UcHost)) - } - for _, host := range ucHosts { - if len(host) > 0 { - ucHosts = append(ucHosts, endpoint(useHttps, host)) +func getUcEndpoint(useHttps bool, hosts []string) region_v2.EndpointsProvider { + if len(hosts) == 0 { + if len(UcHost) > 0 { + hosts = append(hosts, endpoint(useHttps, UcHost)) + } + + for _, host := range ucHosts { + if len(host) > 0 { + hosts = append(hosts, endpoint(useHttps, host)) + } } } - if len(ucHosts) > 0 { - return region_v2.Endpoints{Preferred: ucHosts} + + if len(hosts) > 0 { + return region_v2.Endpoints{Preferred: hosts} } else { return nil } @@ -347,7 +350,12 @@ func GetRegionsInfo(mac *auth.Credentials) ([]RegionInfo, error) { } func GetRegionsInfoWithOptions(mac *auth.Credentials, options UCApiOptions) ([]RegionInfo, error) { + var httpClient clientv2.Client + if options.Client != nil { + httpClient = options.Client.Client + } response, err := apis.NewStorage(&http_client.Options{ + BasicHTTPClient: httpClient, HostFreezeDuration: options.HostFreezeDuration, HostRetryConfig: &clientv2.RetryConfig{ RetryMax: options.RetryMax, @@ -355,11 +363,12 @@ func GetRegionsInfoWithOptions(mac *auth.Credentials, options UCApiOptions) ([]R }).GetRegions( context.Background(), &apis.GetRegionsRequest{Credentials: mac}, - &apis.Options{OverwrittenBucketHosts: getUcEndpoint(options.UseHttps)}, + &apis.Options{OverwrittenBucketHosts: getUcEndpoint(options.UseHttps, options.Hosts)}, ) if err != nil { return nil, err } + regions := make([]RegionInfo, 0, len(response.Regions)) for _, region := range response.Regions { regions = append(regions, RegionInfo{ @@ -377,6 +386,9 @@ type ucClientConfig struct { // 单域名重试次数 RetryMax int + // 请求的域名 + Hosts []string + // 主备域名冻结时间（默认：600s），当一个域名请求失败（单个域名会被重试 TryTimes 次），会被冻结一段时间，使用备用域名进行重试，在冻结时间内，域名不能被使用，当一个操作中所有域名竣备冻结操作不在进行重试，返回最后一次操作的错误。 HostFreezeDuration time.Duration @@ -384,7 +396,11 @@ type ucClientConfig struct { } func getUCClient(config ucClientConfig, mac *auth.Credentials) clientv2.Client { - allHosts := getUcBackupHosts() + allHosts := config.Hosts + if len(allHosts) == 0 { + allHosts = getUcBackupHosts() + } + var hosts []string = nil if !config.IsUcQueryApi { // 非 uc query api 去除 defaultApiHost diff --git a/storage/region_test.go b/storage/region_test.go index 703e9806..247964dd 100644 --- a/storage/region_test.go +++ b/storage/region_test.go @@ -5,9 +5,10 @@ package storage import ( "encoding/json" - "github.com/qiniu/go-sdk/v7/client" "strings" "testing" + + "github.com/qiniu/go-sdk/v7/client" ) func TestRegion(t *testing.T) { @@ -150,6 +151,17 @@ func TestRegionWithSetHost(t *testing.T) { if !strings.HasPrefix(region1.IovipHost, "iovip") || !strings.HasSuffix(region1.IovipHost, ".qbox.me") { t.Fatalf("region1.IovipHost is wrong: %v\v", region1.IovipHost) } + + region1, err = GetRegionWithOptions(testAK, testBucket, UCApiOptions{ + UseHttps: true, + RetryMax: 0, + Hosts: []string{"mock.uc.com"}, + HostFreezeDuration: 0, + Client: nil, + }) + if err == nil { + t.Fatalf("request should be wrong") + } } func TestRegionV4(t *testing.T) { @@ -161,6 +173,17 @@ func TestRegionV4(t *testing.T) { if len(regionGroup.regions) == 0 { t.Fatalf("region1.IovipHost is wrong") } + + _, err = getRegionGroupWithOptions(testAK, testBucket, UCApiOptions{ + UseHttps: true, + RetryMax: 0, + Hosts: []string{"mock.uc.com"}, + HostFreezeDuration: 0, + Client: nil, + }) + if err == nil { + t.Fatalf("request should be wrong") + } } func TestRegionV4WithNoProtocol(t *testing.T) { diff --git a/storage/region_uc_v2.go b/storage/region_uc_v2.go index 9e6678a1..05596a56 100644 --- a/storage/region_uc_v2.go +++ b/storage/region_uc_v2.go @@ -12,6 +12,7 @@ import ( "golang.org/x/sync/singleflight" + "github.com/qiniu/go-sdk/v7/client" "github.com/qiniu/go-sdk/v7/internal/clientv2" ) @@ -209,9 +210,28 @@ func storeRegionV2Cache() { type UCApiOptions struct { UseHttps bool // - RetryMax int // 单域名重试次数 + + RetryMax int // 单域名重试次数 + + Hosts []string // api 请求的域名 + // 主备域名冻结时间（默认：600s），当一个域名请求失败（单个域名会被重试 TryTimes 次），会被冻结一段时间，使用备用域名进行重试，在冻结时间内，域名不能被使用，当一个操作中所有域名竣备冻结操作不在进行重试，返回最后一次操作的错误。 HostFreezeDuration time.Duration + + Client *client.Client // api 请求使用的 client +} + +func (o *UCApiOptions) init() { + if len(o.Hosts) == 0 { + o.Hosts = getUcBackupHosts() + } +} + +func (o *UCApiOptions) firstHost() string { + if len(o.Hosts) == 0 { + return "" + } + return o.Hosts[0] } func DefaultUCApiOptions() UCApiOptions { @@ -223,6 +243,7 @@ func DefaultUCApiOptions() UCApiOptions { } func getRegionByV2(ak, bucket string, options UCApiOptions) (*Region, error) { + options.init() regionV2CacheLock.RLock() if regionV2CacheLoaded { @@ -240,20 +261,22 @@ func getRegionByV2(ak, bucket string, options UCApiOptions) (*Region, error) { }() } - regionCacheKey := makeRegionCacheKey(ak, bucket) + regionCacheKey := makeRegionCacheKey(ak, bucket, options.Hosts) //check from cache if v, ok := regionV2Cache.Load(regionCacheKey); ok && time.Now().Before(v.(regionV2CacheValue).Deadline) { return v.(regionV2CacheValue).Region, nil } newRegion, err, _ := ucQueryV2Group.Do(regionCacheKey, func() (interface{}, error) { - reqURL := fmt.Sprintf("%s/v2/query?ak=%s&bucket=%s", getUcHost(options.UseHttps), ak, bucket) + reqURL := fmt.Sprintf("%s/v2/query?ak=%s&bucket=%s", endpoint(options.UseHttps, options.firstHost()), ak, bucket) var ret UcQueryRet c := getUCClient(ucClientConfig{ IsUcQueryApi: true, RetryMax: options.RetryMax, + Hosts: options.Hosts, HostFreezeDuration: options.HostFreezeDuration, + Client: options.Client, }, nil) err := clientv2.DoAndDecodeJsonResponse(c, clientv2.RequestParams{ Context: context.Background(), @@ -294,6 +317,7 @@ func getRegionByV2(ak, bucket string, options UCApiOptions) (*Region, error) { return newRegion.(*Region), err } -func makeRegionCacheKey(ak, bucket string) string { - return fmt.Sprintf("%s:%s:%x", ak, bucket, md5.Sum([]byte(getUcHost(false)))) +func makeRegionCacheKey(ak, bucket string, ucHosts []string) string { + hostStrings := fmt.Sprintf("%v", ucHosts) + return fmt.Sprintf("%s:%s:%x", ak, bucket, md5.Sum([]byte(hostStrings))) } diff --git a/storage/region_uc_v4.go b/storage/region_uc_v4.go index 0f8bac4e..9e059b8f 100644 --- a/storage/region_uc_v4.go +++ b/storage/region_uc_v4.go @@ -123,6 +123,8 @@ func storeRegionV4Cache() { } func getRegionByV4(ak, bucket string, options UCApiOptions) (*RegionGroup, error) { + options.init() + regionV4CacheLock.RLock() if regionV4CacheLoaded { regionV4CacheLock.RUnlock() @@ -139,7 +141,7 @@ func getRegionByV4(ak, bucket string, options UCApiOptions) (*RegionGroup, error }() } - regionCacheKey := makeRegionCacheKey(ak, bucket) + regionCacheKey := makeRegionCacheKey(ak, bucket, options.Hosts) //check from cache if v, ok := regionV4Cache.Load(regionCacheKey); ok && time.Now().Before(v.(regionV4CacheValue).Deadline) { cacheValue, _ := v.(regionV4CacheValue) @@ -147,13 +149,15 @@ func getRegionByV4(ak, bucket string, options UCApiOptions) (*RegionGroup, error } newRegion, err, _ := ucQueryV4Group.Do(regionCacheKey, func() (interface{}, error) { - reqURL := fmt.Sprintf("%s/v4/query?ak=%s&bucket=%s", getUcHost(options.UseHttps), ak, bucket) + reqURL := fmt.Sprintf("%s/v4/query?ak=%s&bucket=%s", endpoint(options.UseHttps, options.firstHost()), ak, bucket) var ret ucQueryV4Ret c := getUCClient(ucClientConfig{ IsUcQueryApi: true, RetryMax: options.RetryMax, + Hosts: options.Hosts, HostFreezeDuration: options.HostFreezeDuration, + Client: options.Client, }, nil) err := clientv2.DoAndDecodeJsonResponse(c, clientv2.RequestParams{ Context: context.Background(),