Get region support specified UC hosts and Client & List bucket suppor…

…t output file parts (#125)

CHANGELOG.md

@@ -1,4 +1,9 @@
 # Changelog
+## 7.20.1
+* 新增
+  * 获取区域 API 支持单独配置 UC 域名
+  * BucketManager List Bucket 接口支持返回文件的 parts
+
 ## 7.20.0
 * 新增
   * 新版存储客户端库 storagev2 包，包含

README.md

@@ -17,7 +17,7 @@ github.com/qiniu/go-sdk
 在您的项目中的 `go.mod` 文件内添加这行代码
 
 ```
-require github.com/qiniu/go-sdk/v7 v7.20.0
+require github.com/qiniu/go-sdk/v7 v7.20.1
 ```
 
 并且在项目中使用 `"github.com/qiniu/go-sdk/v7"` 引用 Qiniu Go SDK。

conf/conf.go

@@ -5,7 +5,7 @@ import (
 	"strings"
 )
 
-const Version = "7.20.0"
+const Version = "7.20.1"
 
 const (
 	CONTENT_TYPE_JSON      = "application/json"

storage/bucket.go

@@ -1023,7 +1023,7 @@ func (m *BucketManager) Zone(bucket string) (z *Zone, err error) {
 }
 
 func (m *BucketManager) makeRequestOptions() *apis.Options {
-	return &apis.Options{OverwrittenBucketHosts: getUcEndpoint(m.Cfg.UseHTTPS)}
+	return &apis.Options{OverwrittenBucketHosts: getUcEndpoint(m.Cfg.UseHTTPS, nil)}
 }
 
 // 构建op的方法，导出的方法支持在Batch操作中使用

storage/bucket_list.go

@@ -54,6 +54,11 @@ type ListItem struct {
 	 * 文件的 md5 值
 	 */
 	Md5 string `json:"md5"`
+
+	/**
+	 * 文件的分片信息
+	 */
+	Parts []uint `json:"parts"`
 }
 
 // 接口可能返回空的记录
@@ -103,6 +108,7 @@ type listInputOptions struct {
 	delimiter string
 	marker    string
 	limit     int
+	needParts bool
 }
 
 type ListInputOption func(options *listInputOptions)
@@ -131,6 +137,12 @@ func ListInputOptionsLimit(limit int) ListInputOption {
 	}
 }
 
+func ListInputOptionsNeedParts(needParts bool) ListInputOption {
+	return func(input *listInputOptions) {
+		input.needParts = needParts
+	}
+}
+
 // ListFilesWithContext
 //
 //	 @Description: 用来获取空间文件列表，可以根据需要指定文件的列举条件
@@ -166,7 +178,7 @@ func (m *BucketManager) ListFilesWithContext(ctx context.Context, bucket string,
 	}
 
 	ret = &ListFilesRet{}
-	reqURL := fmt.Sprintf("%s%s", host, uriListFiles(bucket, inputOptions.prefix, inputOptions.delimiter, inputOptions.marker, inputOptions.limit))
+	reqURL := fmt.Sprintf("%s%s", host, uriListFiles(bucket, inputOptions.prefix, inputOptions.delimiter, inputOptions.marker, inputOptions.limit, inputOptions.needParts))
 	err = m.Client.CredentialedCall(ctx, m.Mac, auth.TokenQiniu, ret, "POST", reqURL, nil)
 	if err != nil {
 		return nil, false, err
@@ -228,7 +240,7 @@ func (m *BucketManager) ListBucketContext(ctx context.Context, bucket, prefix, d
 	return retCh, err
 }
 
-func uriListFiles(bucket, prefix, delimiter, marker string, limit int) string {
+func uriListFiles(bucket, prefix, delimiter, marker string, limit int, needParts bool) string {
 	query := make(url.Values)
 	query.Add("bucket", bucket)
 	if prefix != "" {
@@ -243,5 +255,6 @@ func uriListFiles(bucket, prefix, delimiter, marker string, limit int) string {
 	if limit > 0 {
 		query.Add("limit", strconv.FormatInt(int64(limit), 10))
 	}
+	query.Add("needparts", strconv.FormatBool(needParts))
 	return fmt.Sprintf("/list?%s", query.Encode())
 }

storage/bucket_list_test.go

@@ -0,0 +1,47 @@
+//go:build integration
+// +build integration
+
+package storage
+
+import (
+	"context"
+	"testing"
+)
+
+func TestList(t *testing.T) {
+	ret, _, err := bucketManager.ListFilesWithContext(context.Background(), testBucket,
+		ListInputOptionsLimit(1000),
+		ListInputOptionsNeedParts(false),
+	)
+	if err != nil {
+		t.Fatalf("List bucket files error: %v\n", err)
+	}
+
+	hasParts := false
+	for _, item := range ret.Items {
+		if len(item.Parts) > 0 {
+			hasParts = true
+		}
+	}
+	if hasParts {
+		t.Fatal("list files: should no parts")
+	}
+
+	ret, _, err = bucketManager.ListFilesWithContext(context.Background(), testBucket,
+		ListInputOptionsLimit(1000),
+		ListInputOptionsNeedParts(true),
+	)
+	if err != nil {
+		t.Fatalf("List bucket files error: %v\n", err)
+	}
+
+	hasParts = false
+	for _, item := range ret.Items {
+		if len(item.Parts) > 0 {
+			hasParts = true
+		}
+	}
+	if !hasParts {
+		t.Fatal("list files: should parts")
+	}
+}

storage/region.go

@@ -290,18 +290,21 @@ func getUcBackupHosts() []string {
 	return hosts
 }
 
-func getUcEndpoint(useHttps bool) region_v2.EndpointsProvider {
-	ucHosts := make([]string, 0, 1+len(ucHosts))
-	if len(UcHost) > 0 {
-		ucHosts = append(ucHosts, endpoint(useHttps, UcHost))
-	}
-	for _, host := range ucHosts {
-		if len(host) > 0 {
-			ucHosts = append(ucHosts, endpoint(useHttps, host))
+func getUcEndpoint(useHttps bool, hosts []string) region_v2.EndpointsProvider {
+	if len(hosts) == 0 {
+		if len(UcHost) > 0 {
+			hosts = append(hosts, endpoint(useHttps, UcHost))
+		}
+
+		for _, host := range ucHosts {
+			if len(host) > 0 {
+				hosts = append(hosts, endpoint(useHttps, host))
+			}
 		}
 	}
-	if len(ucHosts) > 0 {
-		return region_v2.Endpoints{Preferred: ucHosts}
+
+	if len(hosts) > 0 {
+		return region_v2.Endpoints{Preferred: hosts}
 	} else {
 		return nil
 	}
@@ -347,19 +350,25 @@ func GetRegionsInfo(mac *auth.Credentials) ([]RegionInfo, error) {
 }
 
 func GetRegionsInfoWithOptions(mac *auth.Credentials, options UCApiOptions) ([]RegionInfo, error) {
+	var httpClient clientv2.Client
+	if options.Client != nil {
+		httpClient = options.Client.Client
+	}
 	response, err := apis.NewStorage(&http_client.Options{
+		BasicHTTPClient:    httpClient,
 		HostFreezeDuration: options.HostFreezeDuration,
 		HostRetryConfig: &clientv2.RetryConfig{
 			RetryMax: options.RetryMax,
 		},
 	}).GetRegions(
 		context.Background(),
 		&apis.GetRegionsRequest{Credentials: mac},
-		&apis.Options{OverwrittenBucketHosts: getUcEndpoint(options.UseHttps)},
+		&apis.Options{OverwrittenBucketHosts: getUcEndpoint(options.UseHttps, options.Hosts)},
 	)
 	if err != nil {
 		return nil, err
 	}
+
 	regions := make([]RegionInfo, 0, len(response.Regions))
 	for _, region := range response.Regions {
 		regions = append(regions, RegionInfo{
@@ -377,14 +386,21 @@ type ucClientConfig struct {
 	// 单域名重试次数
 	RetryMax int
 
+	// 请求的域名
+	Hosts []string
+
 	// 主备域名冻结时间（默认：600s），当一个域名请求失败（单个域名会被重试 TryTimes 次），会被冻结一段时间，使用备用域名进行重试，在冻结时间内，域名不能被使用，当一个操作中所有域名竣备冻结操作不在进行重试，返回最后一次操作的错误。
 	HostFreezeDuration time.Duration
 
 	Client *client.Client
 }
 
 func getUCClient(config ucClientConfig, mac *auth.Credentials) clientv2.Client {
-	allHosts := getUcBackupHosts()
+	allHosts := config.Hosts
+	if len(allHosts) == 0 {
+		allHosts = getUcBackupHosts()
+	}
+
 	var hosts []string = nil
 	if !config.IsUcQueryApi {
 		// 非 uc query api 去除 defaultApiHost

storage/region_test.go

@@ -5,9 +5,10 @@ package storage
 
 import (
 	"encoding/json"
-	"github.com/qiniu/go-sdk/v7/client"
 	"strings"
 	"testing"
+
+	"github.com/qiniu/go-sdk/v7/client"
 )
 
 func TestRegion(t *testing.T) {
@@ -150,6 +151,17 @@ func TestRegionWithSetHost(t *testing.T) {
 	if !strings.HasPrefix(region1.IovipHost, "iovip") || !strings.HasSuffix(region1.IovipHost, ".qbox.me") {
 		t.Fatalf("region1.IovipHost is wrong: %v\v", region1.IovipHost)
 	}
+
+	region1, err = GetRegionWithOptions(testAK, testBucket, UCApiOptions{
+		UseHttps:           true,
+		RetryMax:           0,
+		Hosts:              []string{"mock.uc.com"},
+		HostFreezeDuration: 0,
+		Client:             nil,
+	})
+	if err == nil {
+		t.Fatalf("request should be wrong")
+	}
 }
 
 func TestRegionV4(t *testing.T) {
@@ -161,6 +173,17 @@ func TestRegionV4(t *testing.T) {
 	if len(regionGroup.regions) == 0 {
 		t.Fatalf("region1.IovipHost is wrong")
 	}
+
+	_, err = getRegionGroupWithOptions(testAK, testBucket, UCApiOptions{
+		UseHttps:           true,
+		RetryMax:           0,
+		Hosts:              []string{"mock.uc.com"},
+		HostFreezeDuration: 0,
+		Client:             nil,
+	})
+	if err == nil {
+		t.Fatalf("request should be wrong")
+	}
 }
 
 func TestRegionV4WithNoProtocol(t *testing.T) {

storage/region_uc_v2.go

@@ -12,6 +12,7 @@ import (
 
 	"golang.org/x/sync/singleflight"
 
+	"github.com/qiniu/go-sdk/v7/client"
 	"github.com/qiniu/go-sdk/v7/internal/clientv2"
 )
 
@@ -209,9 +210,28 @@ func storeRegionV2Cache() {
 
 type UCApiOptions struct {
 	UseHttps bool //
-	RetryMax int  // 单域名重试次数
+
+	RetryMax int // 单域名重试次数
+
+	Hosts []string // api 请求的域名
+
 	// 主备域名冻结时间（默认：600s），当一个域名请求失败（单个域名会被重试 TryTimes 次），会被冻结一段时间，使用备用域名进行重试，在冻结时间内，域名不能被使用，当一个操作中所有域名竣备冻结操作不在进行重试，返回最后一次操作的错误。
 	HostFreezeDuration time.Duration
+
+	Client *client.Client // api 请求使用的 client
+}
+
+func (o *UCApiOptions) init() {
+	if len(o.Hosts) == 0 {
+		o.Hosts = getUcBackupHosts()
+	}
+}
+
+func (o *UCApiOptions) firstHost() string {
+	if len(o.Hosts) == 0 {
+		return ""
+	}
+	return o.Hosts[0]
 }
 
 func DefaultUCApiOptions() UCApiOptions {
@@ -223,6 +243,7 @@ func DefaultUCApiOptions() UCApiOptions {
 }
 
 func getRegionByV2(ak, bucket string, options UCApiOptions) (*Region, error) {
+	options.init()
 
 	regionV2CacheLock.RLock()
 	if regionV2CacheLoaded {
@@ -240,20 +261,22 @@ func getRegionByV2(ak, bucket string, options UCApiOptions) (*Region, error) {
 		}()
 	}
 
-	regionCacheKey := makeRegionCacheKey(ak, bucket)
+	regionCacheKey := makeRegionCacheKey(ak, bucket, options.Hosts)
 	//check from cache
 	if v, ok := regionV2Cache.Load(regionCacheKey); ok && time.Now().Before(v.(regionV2CacheValue).Deadline) {
 		return v.(regionV2CacheValue).Region, nil
 	}
 
 	newRegion, err, _ := ucQueryV2Group.Do(regionCacheKey, func() (interface{}, error) {
-		reqURL := fmt.Sprintf("%s/v2/query?ak=%s&bucket=%s", getUcHost(options.UseHttps), ak, bucket)
+		reqURL := fmt.Sprintf("%s/v2/query?ak=%s&bucket=%s", endpoint(options.UseHttps, options.firstHost()), ak, bucket)
 
 		var ret UcQueryRet
 		c := getUCClient(ucClientConfig{
 			IsUcQueryApi:       true,
 			RetryMax:           options.RetryMax,
+			Hosts:              options.Hosts,
 			HostFreezeDuration: options.HostFreezeDuration,
+			Client:             options.Client,
 		}, nil)
 		err := clientv2.DoAndDecodeJsonResponse(c, clientv2.RequestParams{
 			Context: context.Background(),
@@ -294,6 +317,7 @@ func getRegionByV2(ak, bucket string, options UCApiOptions) (*Region, error) {
 	return newRegion.(*Region), err
 }
 
-func makeRegionCacheKey(ak, bucket string) string {
-	return fmt.Sprintf("%s:%s:%x", ak, bucket, md5.Sum([]byte(getUcHost(false))))
+func makeRegionCacheKey(ak, bucket string, ucHosts []string) string {
+	hostStrings := fmt.Sprintf("%v", ucHosts)
+	return fmt.Sprintf("%s:%s:%x", ak, bucket, md5.Sum([]byte(hostStrings)))
 }

storage/region_uc_v4.go

@@ -123,6 +123,8 @@ func storeRegionV4Cache() {
 }
 
 func getRegionByV4(ak, bucket string, options UCApiOptions) (*RegionGroup, error) {
+	options.init()
+
 	regionV4CacheLock.RLock()
 	if regionV4CacheLoaded {
 		regionV4CacheLock.RUnlock()
@@ -139,21 +141,23 @@ func getRegionByV4(ak, bucket string, options UCApiOptions) (*RegionGroup, error
 		}()
 	}
 
-	regionCacheKey := makeRegionCacheKey(ak, bucket)
+	regionCacheKey := makeRegionCacheKey(ak, bucket, options.Hosts)
 	//check from cache
 	if v, ok := regionV4Cache.Load(regionCacheKey); ok && time.Now().Before(v.(regionV4CacheValue).Deadline) {
 		cacheValue, _ := v.(regionV4CacheValue)
 		return NewRegionGroup(cacheValue.getRegions()...), nil
 	}
 
 	newRegion, err, _ := ucQueryV4Group.Do(regionCacheKey, func() (interface{}, error) {
-		reqURL := fmt.Sprintf("%s/v4/query?ak=%s&bucket=%s", getUcHost(options.UseHttps), ak, bucket)
+		reqURL := fmt.Sprintf("%s/v4/query?ak=%s&bucket=%s", endpoint(options.UseHttps, options.firstHost()), ak, bucket)
 
 		var ret ucQueryV4Ret
 		c := getUCClient(ucClientConfig{
 			IsUcQueryApi:       true,
 			RetryMax:           options.RetryMax,
+			Hosts:              options.Hosts,
 			HostFreezeDuration: options.HostFreezeDuration,
+			Client:             options.Client,
 		}, nil)
 		err := clientv2.DoAndDecodeJsonResponse(c, clientv2.RequestParams{
 			Context: context.Background(),