diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml new file mode 100644 index 0000000..fb16a70 --- /dev/null +++ b/.github/workflows/ci-test.yml @@ -0,0 +1,34 @@ +name: tests +on: [push, pull_request] +jobs: + build: + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest] + go-version: [1.21.x] + include: + - go-version: 1.20.x + os: ubuntu-latest + runs-on: ${{ matrix.os }} + steps: + - name: Install Go + uses: actions/setup-go@v2 + with: + go-version: ${{ matrix.go-version }} + - name: Checkout code + uses: actions/checkout@v2 + with: + ref: ${{ github.ref }} + - name: Format + run: if [ "$(gofmt -s -l . | wc -l)" -gt 0 ]; then exit 1; fi + - name: Golint + run: | + set -e + go install honnef.co/go/tools/cmd/staticcheck@latest + staticcheck -go 1.20 ./... + - name: Run Tests + run: | + bash -e + go test -v ./... + shell: bash diff --git a/README.md b/README.md index 16be02c..cda366d 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,200 @@ -# go-cloud-dev-blob-qiniu-driver -Qiniu gocloud.dev blob driver +# Qiniu gocloud.dev blob driver + +七牛 kodoblob 为 [gocloud.dev](https://gocloud.dev/) 提供了驱动，可以通过使用 [blob](https://gocloud.dev/blob) 包对七牛 Bucket 中的 Blob 进行读写，列举或删除。 + +## 代码案例 + +### 打开一个七牛 Bucket + +```go +package main + +import ( + "context" + "fmt" + "os" + + _ "github.com/bachue/go-cloud-dev-qiniu-driver/kodoblob" + "gocloud.dev/blob" +) + +func main() { + bucket, err := blob.OpenBucket(context.Background(), "kodo://:@?useHttps") + if err != nil { + fmt.Fprintf(os.Stderr, "could not open bucket: %v\n", err) + os.Exit(1) + } + defer bucket.Close() + + // 对 bucket 进行操作 +} +``` + +这里的 URL 必须遵循以下格式 + +``` +kodo://:@? +``` + +其中 `Options` 以 URL 查询的形式设置，支持以下选项： + +| 名称 | 值类型 | 备注 | +|---|---|---| +| `useHttps` | 布尔值 | 是否使用 HTTPS 协议，默认不使用 | +| `downloadDomain` | 字符串列表 | 下载域名，如果不配置则使用默认源站域名，可以配置多个下载域名 | +| `signDownloadUrl` | 布尔值 | 是否对下载 URL 签名，对于私有空间来说，这是必须的，默认不签名 | +| `bucketHost` | 字符串列表 | 设置 Bucket 域名，可以配置多个 Bucket 域名，默认使用公有云 Bucket 域名 | +| `srcUpHost` | 字符串列表 | 设置上传源站域名，可以配置多个上传源站域名，默认通过 Bucket 域名查询获取 | +| `cdnUpHost` | 字符串列表 | 设置上传加速域名，可以配置多个上传加速域名，默认通过 Bucket 域名查询获取 | +| `rsHost` | 字符串列表 | 设置 RS 域名，可以配置多个 RS 域名，默认通过 Bucket 域名查询获取 | +| `rsfHost` | 字符串列表 | 设置 RSF 域名，可以配置多个 RSF 域名，默认通过 Bucket 域名查询获取 | +| `apiHost` | 字符串列表 | 设置 API 域名，可以配置多个 API 域名，默认通过 Bucket 域名查询获取 | + +### 向七牛 Bucket 写入数据 + +```go +package main + +import ( + "context" + "fmt" + "os" + + _ "github.com/bachue/go-cloud-dev-qiniu-driver/kodoblob" + "gocloud.dev/blob" +) + +func main() { + bucket, err := blob.OpenBucket(context.Background(), "kodo://:@?useHttps") + if err != nil { + fmt.Fprintf(os.Stderr, "could not open bucket: %v\n", err) + os.Exit(1) + } + defer bucket.Close() + + w, err := bucket.NewWriter(context.Background(), "", nil) + if err != nil { + fmt.Fprintf(os.Stderr, "could not open object for writing: %v\n", err) + os.Exit(1) + } + defer w.Close() + + // 对 w 写入数据 +} + +``` + +### 从七牛 Bucket 读取数据 + +```go +package main + +import ( + "context" + "fmt" + "os" + + _ "github.com/bachue/go-cloud-dev-qiniu-driver/kodoblob" + "gocloud.dev/blob" +) + +func main() { + bucket, err := blob.OpenBucket(context.Background(), "kodo://:@?useHttps") + if err != nil { + fmt.Fprintf(os.Stderr, "could not open bucket: %v\n", err) + os.Exit(1) + } + defer bucket.Close() + + r, err := bucket.NewReader(context.Background(), "", nil) + if err != nil { + fmt.Fprintf(os.Stderr, "could not open object for reading: %v\n", err) + os.Exit(1) + } + defer r.Close() + + // 从 r 读取数据 +} +``` + +### 从七牛 Bucket 读取范围数据 + +`gocloud.dev/blob` 支持读取指定偏移量的数据。 + +```go +package main + +import ( + "context" + "fmt" + "os" + + _ "github.com/bachue/go-cloud-dev-qiniu-driver/kodoblob" + "gocloud.dev/blob" +) + +func main() { + bucket, err := blob.OpenBucket(context.Background(), "kodo://:@?useHttps") + if err != nil { + fmt.Fprintf(os.Stderr, "could not open bucket: %v\n", err) + os.Exit(1) + } + defer bucket.Close() + + r, err := bucket.NewRangeReader(ctx, "", 1024, 4096, nil) + if err != nil { + fmt.Fprintf(os.Stderr, "could not open object for reading: %v\n", err) + os.Exit(1) + } + defer r.Close() + + // 从 r 读取数据 +} +``` + +### 从七牛 Bucket 删除数据 + +```go +package main + +import ( + "context" + "fmt" + "os" + + _ "github.com/bachue/go-cloud-dev-qiniu-driver/kodoblob" + "gocloud.dev/blob" +) + +func main() { + bucket, err := blob.OpenBucket(context.Background(), "kodo://:@?useHttps") + if err != nil { + fmt.Fprintf(os.Stderr, "could not open bucket: %v\n", err) + os.Exit(1) + } + defer bucket.Close() + + if err = bucket.Delete(context.Background(), "1G.65"); err != nil { + fmt.Fprintf(os.Stderr, "could not delete object: %v\n", err) + os.Exit(1) + } +} +``` + +## 贡献记录 + +- [所有贡献者](https://github.com/bachue/go-cloud-dev-qiniu-driver/contributors) + +## 联系我们 + +- 如果需要帮助，请提交工单（在portal右侧点击咨询和建议提交工单，或者直接向 support@qiniu.com 发送邮件） +- 如果有什么问题，可以到问答社区提问，[问答社区](http://qiniu.segmentfault.com/) +- 更详细的文档，见[官方文档站](http://developer.qiniu.com/) +- 如果发现了bug， 欢迎提交 [issue](https://github.com/bachue/go-cloud-dev-qiniu-driver/issues) +- 如果有功能需求，欢迎提交 [issue](https://github.com/bachue/go-cloud-dev-qiniu-driver/issues) +- 如果要提交代码，欢迎提交 [pull request](https://github.com/bachue/go-cloud-dev-qiniu-driver/pulls) +- 欢迎关注我们的[微信](http://www.qiniu.com/#weixin) [微博](http://weibo.com/qiniutek)，及时获取动态信息。 + +## 代码许可 + +The Apache License v2.0. 详情见 [License 文件](https://github.com/bachue/go-cloud-dev-qiniu-driver/blob/master/LICENSE). diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..1071e68 --- /dev/null +++ b/go.mod @@ -0,0 +1,32 @@ +module github.com/bachue/go-cloud-dev-qiniu-driver + +go 1.21 + +require ( + github.com/onsi/ginkgo/v2 v2.12.0 + github.com/onsi/gomega v1.27.10 + github.com/qiniu/go-sdk/v7 v7.18.0 + gocloud.dev v0.34.0 +) + +require ( + github.com/go-logr/logr v1.2.4 // indirect + github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/protobuf v1.5.3 // indirect + github.com/google/go-cmp v0.5.9 // indirect + github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 // indirect + github.com/googleapis/gax-go/v2 v2.12.0 // indirect + go.opencensus.io v0.24.0 // indirect + golang.org/x/net v0.14.0 // indirect + golang.org/x/sync v0.3.0 // indirect + golang.org/x/sys v0.11.0 // indirect + golang.org/x/text v0.12.0 // indirect + golang.org/x/tools v0.12.0 // indirect + golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect + google.golang.org/api v0.134.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230731193218-e0aa005b6bdf // indirect + google.golang.org/grpc v1.57.0 // indirect + google.golang.org/protobuf v1.31.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..3b42bba --- /dev/null +++ b/go.sum @@ -0,0 +1,273 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.110.7 h1:rJyC7nWRg2jWGZ4wSJ5nY65GTdYJkg0cd/uXb+ACI6o= +cloud.google.com/go v0.110.7/go.mod h1:+EYjdK8e5RME/VY/qLCAtuyALQ9q67dvuum8i+H5xsI= +cloud.google.com/go/compute v1.23.0 h1:tP41Zoavr8ptEqaW6j+LQOnyBBhO7OkOMAGrgLopTwY= +cloud.google.com/go/compute v1.23.0/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM= +cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= +cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= +cloud.google.com/go/iam v1.1.1 h1:lW7fzj15aVIXYHREOqjRBV9PsH0Z6u8Y46a1YGvQP4Y= +cloud.google.com/go/iam v1.1.1/go.mod h1:A5avdyVL2tCppe4unb0951eI9jreack+RJ0/d+KUZOU= +cloud.google.com/go/storage v1.31.0 h1:+S3LjjEN2zZ+L5hOwj4+1OkGCsLVe0NzpXKQ1pSdTCI= +cloud.google.com/go/storage v1.31.0/go.mod h1:81ams1PrhW16L4kF7qg+4mTq7SRs5HsbDTM0bWvrwJ0= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/aws/aws-sdk-go v1.44.314 h1:d/5Jyk/Fb+PBd/4nzQg0JuC2W4A0knrDIzBgK/ggAow= +github.com/aws/aws-sdk-go v1.44.314/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/aws/aws-sdk-go-v2 v1.20.0 h1:INUDpYLt4oiPOJl0XwZDK2OVAVf0Rzo+MGVTv9f+gy8= +github.com/aws/aws-sdk-go-v2 v1.20.0/go.mod h1:uWOr0m0jDsiWw8nnXiqZ+YG6LdvAlGYDLLf2NmHZoy4= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.11 h1:/MS8AzqYNAhhRNalOmxUvYs8VEbNGifTnzhPFdcRQkQ= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.11/go.mod h1:va22++AdXht4ccO3kH2SHkHHYvZ2G9Utz+CXKmm2CaU= +github.com/aws/aws-sdk-go-v2/config v1.18.32 h1:tqEOvkbTxwEV7hToRcJ1xZRjcATqwDVsWbAscgRKyNI= +github.com/aws/aws-sdk-go-v2/config v1.18.32/go.mod h1:U3ZF0fQRRA4gnbn9GGvOWLoT2EzzZfAWeKwnVrm1rDc= +github.com/aws/aws-sdk-go-v2/credentials v1.13.31 h1:vJyON3lG7R8VOErpJJBclBADiWTwzcwdkQpTKx8D2sk= +github.com/aws/aws-sdk-go-v2/credentials v1.13.31/go.mod h1:T4sESjBtY2lNxLgkIASmeP57b5j7hTQqCbqG0tWnxC4= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.7 h1:X3H6+SU21x+76LRglk21dFRgMTJMa5QcpW+SqUf5BBg= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.7/go.mod h1:3we0V09SwcJBzNlnyovrR2wWJhWmVdqAsmVs4uronv8= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.76 h1:DJ1kHj0GI9BbX+XhF0kHxlzOVjcncmDUXmCvXdbfdAE= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.76/go.mod h1:/AZCdswMSgwpB2yMSFfY5H4pVeBLnCuPehdmO/r3xSM= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.37 h1:zr/gxAZkMcvP71ZhQOcvdm8ReLjFgIXnIn0fw5AM7mo= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.37/go.mod h1:Pdn4j43v49Kk6+82spO3Tu5gSeQXRsxo56ePPQAvFiA= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.31 h1:0HCMIkAkVY9KMgueD8tf4bRTUanzEYvhw7KkPXIMpO0= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.31/go.mod h1:fTJDMe8LOFYtqiFFFeHA+SVMAwqLhoq0kcInYoLa9Js= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.38 h1:+i1DOFrW3YZ3apE45tCal9+aDKK6kNEbW6Ib7e1nFxE= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.38/go.mod h1:1/jLp0OgOaWIetycOmycW+vYTYgTZFPttJQRgsI1PoU= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.0 h1:U5yySdwt2HPo/pnQec04DImLzWORbeWML1fJiLkKruI= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.0/go.mod h1:EhC/83j8/hL/UB1WmExo3gkElaja/KlmZM/gl1rTfjM= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.12 h1:uAiiHnWihGP2rVp64fHwzLDrswGjEjsPszwRYMiYQPU= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.12/go.mod h1:fUTHpOXqRQpXvEpDPSa3zxCc2fnpW6YnBoba+eQr+Bg= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.32 h1:kvN1jPHr9UffqqG3bSgZ8tx4+1zKVHz/Ktw/BwW6hX8= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.32/go.mod h1:QmMEM7es84EUkbYWcpnkx8i5EW2uERPfrTFeOch128Y= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.31 h1:auGDJ0aLZahF5SPvkJ6WcUuX7iQ7kyl2MamV7Tm8QBk= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.31/go.mod h1:3+lloe3sZuBQw1aBc5MyndvodzQlyqCZ7x1QPDHaWP4= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.0 h1:Wgjft9X4W5pMeuqgPCHIQtbZ87wsgom7S5F8obreg+c= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.0/go.mod h1:FWNzS4+zcWAP05IF7TDYTY1ysZAzIvogxWaDT9p8fsA= +github.com/aws/aws-sdk-go-v2/service/s3 v1.38.1 h1:mTgFVlfQT8gikc5+/HwD8UL9jnUro5MGv8n/VEYF12I= +github.com/aws/aws-sdk-go-v2/service/s3 v1.38.1/go.mod h1:6SOWLiobcZZshbmECRTADIRYliPL0etqFSigauQEeT0= +github.com/aws/aws-sdk-go-v2/service/sso v1.13.1 h1:DSNpSbfEgFXRV+IfEcKE5kTbqxm+MeF5WgyeRlsLnHY= +github.com/aws/aws-sdk-go-v2/service/sso v1.13.1/go.mod h1:TC9BubuFMVScIU+TLKamO6VZiYTkYoEHqlSQwAe2omw= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.15.1 h1:hd0SKLMdOL/Sl6Z0np1PX9LeH2gqNtBe0MhTedA8MGI= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.15.1/go.mod h1:XO/VcyoQ8nKyKfFW/3DMsRQXsfh/052tHTWmg3xBXRg= +github.com/aws/aws-sdk-go-v2/service/sts v1.21.1 h1:pAOJj+80tC8sPVgSDHzMYD6KLWsaLQ1kZw31PTeORbs= +github.com/aws/aws-sdk-go-v2/service/sts v1.21.1/go.mod h1:G8SbvL0rFk4WOJroU8tKBczhsbhj2p/YY7qeJezJ3CI= +github.com/aws/smithy-go v1.14.0 h1:+X90sB94fizKjDmwb4vyl2cTTPXTE5E2G/1mjByb0io= +github.com/aws/smithy-go v1.14.0/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= +github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= +github.com/go-playground/locales v0.14.0/go.mod h1:sawfccIbzZTqEDETgFXqTho0QybSa7l++s0DH+LDiLs= +github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= +github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl+lu/H90nyDXpg0fqeB/AQUGNTVA= +github.com/go-playground/validator/v10 v10.8.0/go.mod h1:9JhgTzTaE31GZDpH/HSvHiRJrJ3iKAgqqH0Bl/Ocjdk= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 h1:yAJXTCF9TqKcTiHJAE8dj7HMvPfh66eeA2JYW7eFpSE= +github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/s2a-go v0.1.4 h1:1kZ/sQM3srePvKs3tXAvQzo66XfcReoqFpIpIccE7Oc= +github.com/google/s2a-go v0.1.4/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/wire v0.5.0 h1:I7ELFeVBr3yfPIcc8+MWvrjk+3VjbcSzoXm3JVa+jD8= +github.com/google/wire v0.5.0/go.mod h1:ngWDr9Qvq3yZA10YrxfyGELY/AFWGVpy9c1LTRi1EoU= +github.com/googleapis/enterprise-certificate-proxy v0.2.5 h1:UR4rDjcgpgEnqpIEvkiqTYKBCKLNmlge2eVjoZfySzM= +github.com/googleapis/enterprise-certificate-proxy v0.2.5/go.mod h1:RxW0N9901Cko1VOCW3SXCpWP+mlIEkk2tP7jnHy9a3w= +github.com/googleapis/gax-go/v2 v2.12.0 h1:A+gCJKdRfqXkr+BIRGtZLibNXf0m1f9E4HG56etFpas= +github.com/googleapis/gax-go/v2 v2.12.0/go.mod h1:y+aIqrI5eb1YGMVJfuV3185Ts/D7qKpsEkdD5+I6QGU= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= +github.com/onsi/ginkgo/v2 v2.12.0 h1:UIVDowFPwpg6yMUpPjGkYvf06K3RAiJXUhCxEwQVHRI= +github.com/onsi/ginkgo/v2 v2.12.0/go.mod h1:ZNEzXISYlqpb8S36iN71ifqLi3vVD1rVJGvWRCJOUpQ= +github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= +github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/qiniu/dyn v1.3.0/go.mod h1:E8oERcm8TtwJiZvkQPbcAh0RL8jO1G0VXJMW3FAWdkk= +github.com/qiniu/go-sdk/v7 v7.18.0 h1:rw4DMSQkK6NRa6IeuX32/POv/go0tRviPTVCJSiBNlk= +github.com/qiniu/go-sdk/v7 v7.18.0/go.mod h1:nqoYCNo53ZlGA521RvRethvxUDvXKt4gtYXOwye868w= +github.com/qiniu/x v1.10.5/go.mod h1:03Ni9tj+N2h2aKnAz+6N0Xfl8FwMEDRC2PAlxekASDs= +github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8= +github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +gocloud.dev v0.34.0 h1:LzlQY+4l2cMtuNfwT2ht4+fiXwWf/NmPTnXUlLmGif4= +gocloud.dev v0.34.0/go.mod h1:psKOachbnvY3DAOPbsFVmLIErwsbWPUG2H5i65D38vE= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= +golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk= +golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= +golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14= +golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.10.0 h1:zHCpF2Khkwy4mMB4bv0U37YtJdTGW8jI0glAApi0Kh8= +golang.org/x/oauth2 v0.10.0/go.mod h1:kTpgurOux7LqtuxjuyZa4Gj2gdezIt/jQtGnNFfypQI= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc= +golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.12.0 h1:YW6HUoUmYBpwSgyaGaZq1fHjrBjX1rlpZ54T6mu2kss= +golang.org/x/tools v0.12.0/go.mod h1:Sc0INKfu04TlqNoRA1hgpFZbhYXHPr4V5DzpSBTPqQM= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= +golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= +google.golang.org/api v0.134.0 h1:ktL4Goua+UBgoP1eL1/60LwZJqa1sIzkLmvoR3hR6Gw= +google.golang.org/api v0.134.0/go.mod h1:sjRL3UnjTx5UqNQS9EWr9N8p7xbHpy1k0XGRLCf3Spk= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20230731193218-e0aa005b6bdf h1:v5Cf4E9+6tawYrs/grq1q1hFpGtzlGFzgWHqwt6NFiU= +google.golang.org/genproto v0.0.0-20230731193218-e0aa005b6bdf/go.mod h1:oH/ZOT02u4kWEp7oYBGYFFkCdKS/uYR9Z7+0/xuuFp8= +google.golang.org/genproto/googleapis/api v0.0.0-20230731193218-e0aa005b6bdf h1:xkVZ5FdZJF4U82Q/JS+DcZA83s/GRVL+QrFMlexk9Yo= +google.golang.org/genproto/googleapis/api v0.0.0-20230731193218-e0aa005b6bdf/go.mod h1:5DZzOUPCLYL3mNkQ0ms0F3EuUNZ7py1Bqeq6sxzI7/Q= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230731193218-e0aa005b6bdf h1:guOdSPaeFgN+jEJwTo1dQ71hdBm+yKSCCKuTRkJzcVo= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230731193218-e0aa005b6bdf/go.mod h1:zBEcrKX2ZOcEkHWxBPAIvYUWOKKMIhYcmNiUIu2ji3I= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw= +google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/kodoblob/kodoblob.go b/kodoblob/kodoblob.go new file mode 100644 index 0000000..cb0f636 --- /dev/null +++ b/kodoblob/kodoblob.go @@ -0,0 +1,544 @@ +package kodoblob + +import ( + "context" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "runtime" + "sort" + "strings" + "sync" + "time" + + "github.com/qiniu/go-sdk/v7/auth" + "github.com/qiniu/go-sdk/v7/storage" + "gocloud.dev/blob" + "gocloud.dev/blob/driver" + "gocloud.dev/gcerrors" +) + +// Scheme is the URL scheme s3blob registers its URLOpener under on +// blob.DefaultMux. +const Scheme = "kodo" + +const version = "0.1.0" + +var ( + ErrNoAccessKey = errors.New("no accessKey provided") + ErrNoSecretKey = errors.New("no secretKey provided") + ErrNoDownloadDomain = errors.New("no downloadDomain provided") + ErrNotSupportedSignedPutUrl = errors.New("kodoblob: does not support SignedURL for PUT") + ErrNotSupportedSignedDeleteUrl = errors.New("kodoblob: does not support SignedURL for DELETE") + + appName = fmt.Sprintf("QiniuGoCloudDevKodoBlob/%s", version) + userAgent = fmt.Sprintf("%s (%s; %s; %s) %s", appName, runtime.GOOS, runtime.GOARCH, runtime.Compiler, runtime.Version()) +) + +func init() { + blob.DefaultURLMux().RegisterBucket(Scheme, new(urlSessionOpener)) + storage.SetAppName(appName) +} + +type urlSessionOpener struct{} + +func (o *urlSessionOpener) OpenBucketURL(ctx context.Context, u *url.URL) (*blob.Bucket, error) { + credentials, err := o.createCredentials(u.User) + if err != nil { + return nil, err + } + config, err := o.createConfig(u.Query()) + if err != nil { + return nil, err + } + downloadDomains, err := o.createDownloadDomains(u.Query()) + if err != nil { + return nil, err + } + _, signDownloadUrl := u.Query()["signDownloadUrl"] + _, preferHttps := u.Query()["useHttps"] + uploadConfig := storage.UploadConfig{ + UseHTTPS: preferHttps, + UseCdnDomains: true, + } + return blob.NewBucket(&bucket{ + name: u.Host, + downloadDomains: downloadDomains, + credentials: credentials, + config: config, + willSignDownloadUrl: signDownloadUrl, + preferHttps: preferHttps, + bucketManager: storage.NewBucketManager(credentials, config), + uploadManager: storage.NewUploadManager(&uploadConfig), + }), nil +} + +func (o *urlSessionOpener) createCredentials(userInfo *url.Userinfo) (*auth.Credentials, error) { + accessKey := userInfo.Username() + if accessKey == "" { + return nil, ErrNoAccessKey + } + + secretKey, secretKeySet := userInfo.Password() + if secretKeySet && secretKey == "" { + return nil, ErrNoSecretKey + } + + return &auth.Credentials{ + AccessKey: accessKey, + SecretKey: []byte(secretKey), + }, nil +} + +func (o *urlSessionOpener) createConfig(query url.Values) (*storage.Config, error) { + var ( + config = &storage.Config{UseCdnDomains: true} + region *storage.Region + useRegion = false + ) + if ucHosts, ok := query["bucketHost"]; ok { + storage.SetUcHosts(ucHosts...) + } else if ucHosts, ok = query["ucHost"]; ok { + storage.SetUcHosts(ucHosts...) + } + _, config.UseHTTPS = query["useHttps"] + if srcUpHosts, ok := query["srcUpHost"]; ok { + region.SrcUpHosts = srcUpHosts + useRegion = true + } + if cdnUpHosts, ok := query["cdnUpHost"]; ok { + region.CdnUpHosts = cdnUpHosts + useRegion = true + } + if rsHost := query.Get("rsHost"); rsHost != "" { + region.RsHost = rsHost + useRegion = true + } + if rsfHost := query.Get("rsfHost"); rsfHost != "" { + region.RsfHost = rsfHost + useRegion = true + } + if apiHost := query.Get("apiHost"); apiHost != "" { + region.ApiHost = apiHost + useRegion = true + } + if useRegion { + config.Region = region + } + return config, nil +} + +func (o *urlSessionOpener) createDownloadDomains(query url.Values) ([]*url.URL, error) { + var downloadUrls []*url.URL + + _, useHttps := query["useHttps"] + if downloadDomains, ok := query["downloadDomain"]; ok { + downloadUrls = make([]*url.URL, 0, len(downloadDomains)) + for _, downloadDomain := range downloadDomains { + if !strings.HasPrefix(downloadDomain, "http://") && !strings.HasPrefix(downloadDomain, "https://") { + if useHttps { + downloadDomain = "https://" + downloadDomain + } else { + downloadDomain = "http://" + downloadDomain + } + } + if downloadUrl, err := url.Parse(downloadDomain); err != nil { + return nil, err + } else { + downloadUrls = append(downloadUrls, downloadUrl) + } + } + } + return downloadUrls, nil +} + +type bucket struct { + name string + downloadDomains []*url.URL + willSignDownloadUrl bool + preferHttps bool + credentials *auth.Credentials + config *storage.Config + uploadManager *storage.UploadManager + bucketManager *storage.BucketManager +} + +func (b *bucket) Close() error { + return nil +} + +func (b *bucket) ErrorCode(err error) gcerrors.ErrorCode { + // TODO + return gcerrors.Unknown +} + +const defaultPageSize = 1000 + +func (b *bucket) ListPaged(ctx context.Context, opts *driver.ListOptions) (*driver.ListPage, error) { + var ( + pageSize int + listInputOptions = make([]storage.ListInputOption, 0, 4) + listResult driver.ListPage + ) + + if opts != nil { + if opts.Prefix != "" { + listInputOptions = append(listInputOptions, storage.ListInputOptionsPrefix(opts.Prefix)) + } + if opts.Delimiter != "" { + listInputOptions = append(listInputOptions, storage.ListInputOptionsDelimiter(opts.Delimiter)) + } + if len(opts.PageToken) > 0 { + listInputOptions = append(listInputOptions, storage.ListInputOptionsMarker(string(opts.PageToken))) + } + pageSize = opts.PageSize + } + + if pageSize == 0 { + pageSize = defaultPageSize + } + listInputOptions = append(listInputOptions, storage.ListInputOptionsLimit(pageSize)) + + if listFilesRet, hasNext, err := b.bucketManager.ListFilesWithContext(ctx, b.name, listInputOptions...); err != nil { + return nil, err + } else { + if hasNext { + listResult.NextPageToken = []byte(listFilesRet.Marker) + } + listResult.Objects = make([]*driver.ListObject, 0, len(listFilesRet.CommonPrefixes)+len(listFilesRet.Items)) + for _, commonPrefix := range listFilesRet.CommonPrefixes { + listResult.Objects = append(listResult.Objects, &driver.ListObject{ + Key: commonPrefix, + IsDir: true, + }) + } + for _, item := range listFilesRet.Items { + listResult.Objects = append(listResult.Objects, &driver.ListObject{ + Key: item.Key, + ModTime: time.UnixMicro(item.PutTime * 10), + Size: item.Fsize, + MD5: []byte(item.Md5), + IsDir: false, + }) + } + if len(listResult.Objects) > 0 { + sort.Slice(listResult.Objects, func(i, j int) bool { + return listResult.Objects[i].Key < listResult.Objects[j].Key + }) + } + } + + return &listResult, nil +} + +func (b *bucket) As(i interface{}) bool { + return false +} + +func (b *bucket) ErrorAs(err error, i interface{}) bool { + return errors.As(err, i) +} + +type ErrStatusCode struct{ code int } + +func (err ErrStatusCode) Error() string { + return fmt.Sprintf("kodoblob: unexpected status code %d", err.code) +} + +func (b *bucket) createDownloadRequest(ctx context.Context, method, key, byteRange string, expiry time.Duration) (*http.Request, error) { + if downloadUrl, err := b.signDownloadUrl(key, expiry); err != nil { + return nil, err + } else if request, err := http.NewRequestWithContext(ctx, method, downloadUrl, http.NoBody); err != nil { + return nil, err + } else { + request.Header.Set("User-Agent", userAgent) + if byteRange != "" { + request.Header.Set("Range", byteRange) + } + return request, nil + } +} + +func (b *bucket) Attributes(ctx context.Context, key string) (*driver.Attributes, error) { + if request, err := b.createDownloadRequest(ctx, http.MethodHead, key, "", 3*time.Minute); err != nil { + return nil, err + } else if response, err := http.DefaultClient.Do(request); err != nil { + return nil, err + } else if err = response.Body.Close(); err != nil { + return nil, err + } else { + if response.StatusCode != http.StatusOK { + return nil, ErrStatusCode{code: response.StatusCode} + } + return b.attributes(response) + } +} + +func (b *bucket) attributes(response *http.Response) (*driver.Attributes, error) { + headers := response.Header + attributes := driver.Attributes{ + CacheControl: headers.Get("Cache-Control"), + ContentDisposition: headers.Get("Content-Disposition"), + ContentEncoding: headers.Get("Content-Encoding"), + ContentLanguage: headers.Get("Content-Language"), + ContentType: headers.Get("Content-Type"), + ETag: headers.Get("Etag"), + MD5: []byte(headers.Get("Content-Md5")), + Size: response.ContentLength, + Metadata: make(map[string]string), + } + if lastModified := headers.Get("Last-Modified"); lastModified != "" { + if t, e := time.Parse(time.RFC1123, lastModified); e == nil { + attributes.ModTime = t + } + } + for k, v := range headers { + k = strings.ToLower(k) + if len(v) > 0 && strings.HasPrefix(k, "x-qn-meta-") { + k = strings.TrimPrefix(k, "x-qn-meta-") + attributes.Metadata[k] = v[0] + } + } + return &attributes, nil +} + +type reader struct { + attributes *driver.Attributes + body io.ReadCloser +} + +func (r reader) As(interface{}) bool { + return false +} + +func (r reader) Read(p []byte) (n int, err error) { + return r.body.Read(p) +} + +func (r reader) Close() error { + return r.body.Close() +} + +func (r reader) Attributes() *driver.ReaderAttributes { + return &driver.ReaderAttributes{ + ContentType: r.attributes.ContentType, + ModTime: r.attributes.ModTime, + Size: r.attributes.Size, + } +} + +func (b *bucket) NewRangeReader(ctx context.Context, key string, offset, length int64, opts *driver.ReaderOptions) (driver.Reader, error) { + var byteRange string + + if offset > 0 && length < 0 { + byteRange = fmt.Sprintf("bytes=%d-", offset) + } else if length == 0 { + byteRange = fmt.Sprintf("bytes=%d-%d", offset, offset) + } else if length >= 0 { + byteRange = fmt.Sprintf("bytes=%d-%d", offset, offset+length-1) + } + + request, err := b.createDownloadRequest(ctx, http.MethodGet, key, byteRange, 3*time.Minute) + if err != nil { + return nil, err + } + response, err := http.DefaultClient.Do(request) + if err != nil { + return nil, err + } + if length == 0 { + response.Body.Close() + response.Body = http.NoBody + } + if response.StatusCode != http.StatusOK && response.StatusCode != http.StatusPartialContent { + return nil, ErrStatusCode{code: response.StatusCode} + } + attributes, err := b.attributes(response) + if err != nil { + return nil, err + } + + return reader{attributes: attributes, body: response.Body}, nil +} + +type writer struct { + b *bucket + key string + ctx context.Context + putPolicy storage.PutPolicy + uploadExtra storage.UploadExtra + source storage.UploadSource + wg sync.WaitGroup + err error + pw io.WriteCloser +} + +func (w *writer) Close() error { + if w.pw != nil { + err := w.pw.Close() + if err == nil { + err = w.err + } + w.err = nil + w.source = nil + w.pw = nil + w.wg.Wait() + return err + } else { + return nil + } +} + +func (w *writer) Write(p []byte) (int, error) { + if w.pw == nil { + var err error + pr, pw := io.Pipe() + w.pw = pw + if w.source, err = storage.NewUploadSourceReader(pr, -1); err != nil { + return 0, err + } + w.wg.Add(1) + go func() { + defer w.wg.Done() + + var ret storage.UploadRet + w.err = w.b.uploadManager.Put(w.ctx, &ret, w.putPolicy.UploadToken(w.b.credentials), &w.key, w.source, &w.uploadExtra) + if w.pw != nil { + w.source = nil + w.pw.Close() + w.pw = nil + } else { + w.err = nil + } + }() + } + if w.err != nil { + return 0, w.err + } + return w.pw.Write(p) +} + +func (w *writer) Upload(r io.Reader) error { + if w.pw != nil { + _, err := io.Copy(w.pw, r) + if err != nil { + return err + } + return w.Close() + } else { + r, err := storage.NewUploadSourceReader(r, -1) + if err != nil { + return err + } + var ret storage.UploadRet + return w.b.uploadManager.Put(w.ctx, &ret, w.putPolicy.UploadToken(w.b.credentials), &w.key, r, &w.uploadExtra) + } +} + +func (b *bucket) NewTypedWriter(ctx context.Context, key string, contentType string, opts *driver.WriterOptions) (driver.Writer, error) { + putPolicy := storage.PutPolicy{ + Scope: fmt.Sprintf("%s:%s", b.name, key), + Expires: 24 * 3600, + } + uploadExtra := storage.UploadExtra{ + Params: convertMetadataToParams(opts.Metadata), + MimeType: contentType, + } + return &writer{ + b: b, + key: key, + ctx: ctx, + putPolicy: putPolicy, + uploadExtra: uploadExtra, + source: nil, + err: nil, + pw: nil, + }, nil +} + +func (b *bucket) Copy(ctx context.Context, dstKey, srcKey string, opts *driver.CopyOptions) error { + // TODO: direct ctx supported + c := make(chan error) + go func() { + c <- b.bucketManager.Copy(b.name, srcKey, b.name, dstKey, true) + }() + select { + case err := <-c: + return err + case <-ctx.Done(): + return ctx.Err() + } +} + +func (b *bucket) Delete(ctx context.Context, key string) error { + // TODO: direct ctx supported + c := make(chan error) + go func() { + c <- b.bucketManager.Delete(b.name, key) + }() + select { + case err := <-c: + return err + case <-ctx.Done(): + return ctx.Err() + } +} + +func (b *bucket) SignedURL(ctx context.Context, key string, opts *driver.SignedURLOptions) (string, error) { + switch opts.Method { + case http.MethodGet: + return b.signDownloadUrl(key, opts.Expiry) + case http.MethodPut: + return "", ErrNotSupportedSignedPutUrl + case http.MethodDelete: + return "", ErrNotSupportedSignedDeleteUrl + default: + return "", fmt.Errorf("unsupported Method %q", opts.Method) + } +} + +func (b *bucket) signDownloadUrl(key string, expiry time.Duration) (string, error) { + var ( + downloadUrl *url.URL + signUrl = b.willSignDownloadUrl + ) + if len(b.downloadDomains) > 0 { + downloadUrl = b.downloadDomains[0] + } else { + region, err := storage.GetRegionWithOptions(b.credentials.AccessKey, b.name, storage.UCApiOptions{UseHttps: b.preferHttps}) + if err != nil { + return "", err + } + ioSrcHost := region.IoSrcHost + if ioSrcHost == "" { + return "", ErrNoDownloadDomain + } + signUrl = true + if !strings.HasPrefix(ioSrcHost, "http://") && !strings.HasPrefix(ioSrcHost, "https://") { + if b.preferHttps { + ioSrcHost = "https://" + ioSrcHost + } else { + ioSrcHost = "http://" + ioSrcHost + } + } + downloadUrl, err = url.Parse(ioSrcHost) + if err != nil { + return "", err + } + } + if signUrl { + return storage.MakePrivateURLv2(b.credentials, downloadUrl.String(), key, time.Now().Add(expiry).Unix()), nil + } else { + return storage.MakePublicURLv2(downloadUrl.String(), key), nil + } +} + +func convertMetadataToParams(metadata map[string]string) map[string]string { + params := make(map[string]string, len(metadata)) + for k, v := range metadata { + params["x-qn-meta-"+k] = v + } + return params +} diff --git a/kodoblob/kodoblob_suite_test.go b/kodoblob/kodoblob_suite_test.go new file mode 100644 index 0000000..3375d77 --- /dev/null +++ b/kodoblob/kodoblob_suite_test.go @@ -0,0 +1,13 @@ +package kodoblob_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestKodoBlob(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "KodoBlob Suite") +} diff --git a/kodoblob/kodoblob_test.go b/kodoblob/kodoblob_test.go new file mode 100644 index 0000000..b9eb934 --- /dev/null +++ b/kodoblob/kodoblob_test.go @@ -0,0 +1,450 @@ +package kodoblob_test + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "math/rand" + "net/http" + "net/url" + "os" + "path/filepath" + "strconv" + "strings" + "sync/atomic" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + _ "github.com/bachue/go-cloud-dev-qiniu-driver/kodoblob" + "gocloud.dev/blob" +) + +var _ = Describe("KodoBlob", func() { + const ( + accessKey = "testaccesskey" + secretKey = "testsecretkey" + bucketName = "fakebucketname" + ) + var ( + bucket *blob.Bucket + ucServer *mockServer + ioServer *mockServer + ioSrcServer *mockServer + rsServer *mockServer + rsfServer *mockServer + upServer *mockServer + apiServer *mockServer + ) + newBucket := func() *blob.Bucket { + values := make(url.Values) + values.Set("bucketHost", ucServer.URL()) + bucket, err := blob.OpenBucket(context.Background(), "kodo://"+accessKey+":"+secretKey+"@"+bucketName+"?"+values.Encode()) + Expect(err).NotTo(HaveOccurred()) + return bucket + } + BeforeEach(func() { + os.RemoveAll(filepath.Join(os.TempDir(), "qiniu-golang-sdk")) + ioServer = newMockServer() + ioSrcServer = newMockServer() + rsServer = newMockServer() + rsfServer = newMockServer() + upServer = newMockServer() + apiServer = newMockServer() + ucServer = newMockServerWithMux(func(mux *http.ServeMux) { + mux.HandleFunc("/v2/query", func(w http.ResponseWriter, r *http.Request) { + Expect(r.Method).To(Equal(http.MethodGet)) + Expect(r.URL.Path).To(Equal("/v2/query")) + Expect(r.URL.Query().Get("ak")).To(Equal(accessKey)) + Expect(r.URL.Query().Get("bucket")).To(Equal(bucketName)) + + err := json.NewEncoder(w).Encode(map[string]any{ + "region": "z0", "ttl": 3600, + "io": map[string]map[string][]string{"src": {"main": {ioServer.Host()}}}, + "io_src": map[string]map[string][]string{"src": {"main": {ioSrcServer.Host()}}}, + "up": map[string]map[string][]string{"src": {"main": {upServer.Host()}}}, + "rs": map[string]map[string][]string{"src": {"main": {rsServer.Host()}}}, + "rsf": map[string]map[string][]string{"src": {"main": {rsfServer.Host()}}}, + "uc": map[string]map[string][]string{"src": {"main": {ucServer.Host()}}}, + "api": map[string]map[string][]string{"src": {"main": {apiServer.Host()}}}, + }) + Expect(err).NotTo(HaveOccurred()) + }) + mux.HandleFunc("/v4/query", func(w http.ResponseWriter, r *http.Request) { + Expect(r.Method).To(Equal(http.MethodGet)) + Expect(r.URL.Path).To(Equal("/v4/query")) + Expect(r.URL.Query().Get("ak")).To(Equal(accessKey)) + Expect(r.URL.Query().Get("bucket")).To(Equal(bucketName)) + + err := json.NewEncoder(w).Encode(map[string][]map[string]any{ + "hosts": { + { + "region": "z0", "ttl": 3600, + "io": map[string][]string{"domains": {ioServer.Host()}}, + "io_src": map[string][]string{"domains": {ioSrcServer.Host()}}, + "up": map[string][]string{"domains": {upServer.Host()}}, + "rs": map[string][]string{"domains": {rsServer.Host()}}, + "rsf": map[string][]string{"domains": {rsfServer.Host()}}, + "uc": map[string][]string{"domains": {ucServer.Host()}}, + "api": map[string][]string{"domains": {apiServer.Host()}}, + }, + }, + }) + Expect(err).NotTo(HaveOccurred()) + }) + }, 1) + bucket = newBucket() + }) + AfterEach(func() { + bucket.Close() + ucServer.Close() + apiServer.Close() + rsServer.Close() + rsfServer.Close() + upServer.Close() + ioServer.Close() + ioSrcServer.Close() + }) + + Context("ListFiles", func() { + It("should list all files", func(ctx context.Context) { + rsfServer.SetHandler(func(w http.ResponseWriter, r *http.Request, n uint32) { + Expect(r.Method).To(Equal(http.MethodPost)) + Expect(r.URL.Path).To(Equal("/list")) + Expect(r.URL.Query().Get("bucket")).To(Equal(bucketName)) + Expect(r.URL.Query().Get("limit")).To(Equal("1000")) + + if n > 5 { + Fail("should not list more than 5 times") + } else if n > 0 { + Expect(r.URL.Query().Get("marker")).To(Equal(fmt.Sprintf("marker_%d", n-1))) + } else { + Expect(r.URL.Query().Has("marker")).To(BeFalse()) + } + + items := make([]map[string]any, 0, 1000) + for i := uint32(0); i < 1000; i++ { + items = append(items, map[string]any{ + "key": fmt.Sprintf("data_%05d", n*1000+i), + "hash": fmt.Sprintf("hash_%05d", n*1000+i), + "md5": fmt.Sprintf("md5_%05d", n*1000+i), + "fsize": n*1000 + i, + "mimeType": "text/plain", + "putTime": time.Now().UnixNano() / 100, + }) + } + responseBodyJson := map[string]any{"items": items} + if n < 5 { + responseBodyJson["marker"] = fmt.Sprintf("marker_%d", n) + } + err := json.NewEncoder(w).Encode(responseBodyJson) + Expect(err).NotTo(HaveOccurred()) + }, 6) + listIter := bucket.List(nil) + objectCount := 0 + for { + object, err := listIter.Next(ctx) + if err != nil { + if err == io.EOF { + break + } + Expect(err).NotTo(HaveOccurred()) + } + Expect(object.Key).To(Equal(fmt.Sprintf("data_%05d", objectCount))) + Expect(object.Size).To(Equal(int64(objectCount))) + Expect(object.MD5).To(Equal([]byte(fmt.Sprintf("md5_%05d", objectCount)))) + Expect(object.IsDir).To(BeFalse()) + objectCount += 1 + } + Expect(objectCount).To(Equal(6000)) + }) + + It("should list all files with prefix", func(ctx context.Context) { + rsfServer.SetHandler(func(w http.ResponseWriter, r *http.Request, n uint32) { + Expect(r.Method).To(Equal(http.MethodPost)) + Expect(r.URL.Path).To(Equal("/list")) + Expect(r.URL.Query().Get("bucket")).To(Equal(bucketName)) + Expect(r.URL.Query().Get("limit")).To(Equal("1000")) + Expect(r.URL.Query().Get("prefix")).To(Equal("data/")) + + items := make([]map[string]any, 0, 1000) + for i := uint32(0); i < 1000; i++ { + items = append(items, map[string]any{ + "key": fmt.Sprintf("data_%05d", n*1000+i), + "hash": fmt.Sprintf("hash_%05d", n*1000+i), + "md5": fmt.Sprintf("md5_%05d", n*1000+i), + "fsize": n*1000 + i, + "mimeType": "text/plain", + "putTime": time.Now().UnixNano() / 100, + }) + } + responseBodyJson := map[string][]map[string]any{"items": items} + err := json.NewEncoder(w).Encode(responseBodyJson) + Expect(err).NotTo(HaveOccurred()) + }, 1) + listIter := bucket.List(&blob.ListOptions{Prefix: "data/"}) + objectCount := 0 + for { + _, err := listIter.Next(ctx) + if err != nil { + if err == io.EOF { + break + } + Expect(err).NotTo(HaveOccurred()) + } + objectCount += 1 + } + Expect(objectCount).To(Equal(1000)) + }) + + It("should list all files with delimiter", func(ctx context.Context) { + rsfServer.SetHandler(func(w http.ResponseWriter, r *http.Request, n uint32) { + Expect(r.Method).To(Equal(http.MethodPost)) + Expect(r.URL.Path).To(Equal("/list")) + Expect(r.URL.Query().Get("bucket")).To(Equal(bucketName)) + Expect(r.URL.Query().Get("limit")).To(Equal("1000")) + Expect(r.URL.Query().Get("delimiter")).To(Equal("/")) + + commonPrefixes := make([]string, 0, 1000) + for i := uint32(0); i < 1000; i++ { + commonPrefixes = append(commonPrefixes, fmt.Sprintf("data_%05d/", n*1000+i)) + } + responseBodyJson := map[string]any{"commonPrefixes": commonPrefixes} + err := json.NewEncoder(w).Encode(responseBodyJson) + Expect(err).NotTo(HaveOccurred()) + }, 1) + listIter := bucket.List(&blob.ListOptions{Delimiter: "/"}) + objectCount := 0 + for { + object, err := listIter.Next(ctx) + if err != nil { + if err == io.EOF { + break + } + Expect(err).NotTo(HaveOccurred()) + } + Expect(object.Key).To(Equal(fmt.Sprintf("data_%05d/", objectCount))) + Expect(object.IsDir).To(BeTrue()) + objectCount += 1 + } + Expect(objectCount).To(Equal(1000)) + }) + }) + + Context("Attributes", func() { + It("should get attributes", func(ctx context.Context) { + ioSrcServer.SetHandler(func(w http.ResponseWriter, r *http.Request, n uint32) { + Expect(r.Method).To(Equal(http.MethodHead)) + Expect(r.URL.Path).To(Equal("/existed-file")) + Expect(r.URL.Query().Has("e")).To(BeTrue()) + Expect(r.URL.Query().Has("token")).To(BeTrue()) + + w.Header().Set("Content-Type", "text/plain") + w.Header().Set("Content-Md5", "fakemd5") + w.Header().Set("Etag", "fakeetag") + w.Header().Set("Content-Length", "1024") + w.Header().Set("Last-Modified", time.Now().Format(time.RFC1123)) + w.Header().Set("x-qn-meta-data-a", "value-1") + w.Header().Set("x-qn-meta-data-b", "value-2") + }, 1) + info, err := bucket.Attributes(ctx, "existed-file") + Expect(err).NotTo(HaveOccurred()) + Expect(info.Size).To(Equal(int64(1024))) + Expect(info.ContentType).To(Equal("text/plain")) + Expect(info.MD5).To(Equal([]byte("fakemd5"))) + Expect(info.ETag).To(Equal("fakeetag")) + Expect(info.ModTime).To(BeTemporally("~", time.Now(), 5*time.Second)) + Expect(info.Metadata["data-a"]).To(Equal("value-1")) + Expect(info.Metadata["data-b"]).To(Equal("value-2")) + }) + + It("should not get attributes from non-existed object", func(ctx context.Context) { + ioSrcServer.SetHandler(func(w http.ResponseWriter, r *http.Request, n uint32) { + Expect(r.Method).To(Equal(http.MethodHead)) + Expect(r.URL.Path).To(Equal("/non-existed")) + Expect(r.URL.Query().Has("e")).To(BeTrue()) + Expect(r.URL.Query().Has("token")).To(BeTrue()) + w.WriteHeader(http.StatusNotFound) + }, 1) + _, err := bucket.Attributes(ctx, "non-existed") + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("kodoblob: unexpected status code 404")) + }) + }) + + Context("Download", func() { + It("should download object", func(ctx context.Context) { + ioSrcServer.SetHandler(func(w http.ResponseWriter, r *http.Request, n uint32) { + Expect(r.Method).To(Equal(http.MethodGet)) + Expect(r.URL.Path).To(Equal("/existed-file")) + Expect(r.URL.Query().Has("e")).To(BeTrue()) + Expect(r.URL.Query().Has("token")).To(BeTrue()) + + w.Header().Set("Content-Type", "text/plain") + w.Header().Set("Content-Length", "1024") + w.Header().Set("Last-Modified", time.Now().Format(time.RFC1123)) + + _, err := io.Copy(w, io.LimitReader(rand.New(rand.NewSource(time.Now().UnixNano())), 1024)) + Expect(err).NotTo(HaveOccurred()) + }, 1) + + reader, err := bucket.NewReader(ctx, "existed-file", nil) + Expect(err).NotTo(HaveOccurred()) + defer reader.Close() + + Expect(reader.Size()).To(Equal(int64(1024))) + Expect(reader.ContentType()).To(Equal("text/plain")) + Expect(reader.ModTime()).To(BeTemporally("~", time.Now(), 5*time.Second)) + + n, err := io.Copy(io.Discard, reader) + Expect(err).NotTo(HaveOccurred()) + Expect(n).To(Equal(reader.Size())) + + err = reader.Close() + Expect(err).NotTo(HaveOccurred()) + }) + + It("should download object in range", func(ctx context.Context) { + ioSrcServer.SetHandler(func(w http.ResponseWriter, r *http.Request, n uint32) { + Expect(r.Method).To(Equal(http.MethodGet)) + Expect(r.URL.Path).To(Equal("/existed-file")) + Expect(r.URL.Query().Has("e")).To(BeTrue()) + Expect(r.URL.Query().Has("token")).To(BeTrue()) + + w.Header().Set("Accept-Ranges", "bytes") + w.Header().Set("Content-Transfer-Encoding", "binary") + w.Header().Set("Content-Type", "text/plain") + w.Header().Set("Content-Length", "2048") + w.Header().Set("Content-Range", "bytes 1024-3071/1056964608") + w.Header().Set("Last-Modified", time.Now().Format(time.RFC1123)) + w.WriteHeader(http.StatusPartialContent) + + _, err := io.Copy(w, io.LimitReader(rand.New(rand.NewSource(time.Now().UnixNano())), 2048)) + Expect(err).NotTo(HaveOccurred()) + }, 1) + + reader, err := bucket.NewRangeReader(ctx, "existed-file", 1024, 2*1024, nil) + Expect(err).NotTo(HaveOccurred()) + defer reader.Close() + + Expect(reader.Size()).To(Equal(int64(2048))) + Expect(reader.ContentType()).To(Equal("text/plain")) + Expect(reader.ModTime()).To(BeTemporally("~", time.Now(), 5*time.Second)) + + n, err := io.Copy(io.Discard, reader) + Expect(err).NotTo(HaveOccurred()) + Expect(n).To(Equal(reader.Size())) + + err = reader.Close() + Expect(err).NotTo(HaveOccurred()) + }) + }) + + Context("Upload", func() { + It("should upload object", func(ctx context.Context) { + blocks := [3][]byte{randData(4 * 1024 * 1024), randData(4 * 1024 * 1024), randData(4 * 1024 * 1024)} + upServer.WithMux(func(mux *http.ServeMux) { + bucketNameBase64ed := base64.URLEncoding.EncodeToString([]byte("existed-file")) + pathPrefix := "/buckets/" + bucketName + "/objects/" + bucketNameBase64ed + "/uploads" + called := uint32(0) + mux.HandleFunc(pathPrefix, func(w http.ResponseWriter, r *http.Request) { + n := atomic.AddUint32(&called, 1) - 1 + path := strings.TrimPrefix(r.URL.Path, pathPrefix) + if path == "" || path == "/" { + Expect(r.Method).To(Equal(http.MethodPost)) + w.Header().Set("Content-Type", "application/json") + err := json.NewEncoder(w).Encode(map[string]any{ + "uploadId": "fakeuploadid", + "expireAt": time.Now().Unix(), + }) + Expect(err).NotTo(HaveOccurred()) + return + } + path = strings.TrimPrefix(path, "/fakeuploadid") + if path == "" || path == "/" { + Expect(r.Method).To(Equal(http.MethodPut)) + type UploadPartInfo struct { + Etag string `json:"etag"` + PartNumber int64 `json:"partNumber"` + } + var body struct { + Parts []UploadPartInfo `json:"parts"` + MimeType string `json:"mimeType,omitempty"` + Metadata map[string]string `json:"metadata,omitempty"` + CustomVars map[string]string `json:"customVars,omitempty"` + } + err := json.NewDecoder(r.Body).Decode(&body) + Expect(err).NotTo(HaveOccurred()) + Expect(body.MimeType).To(Equal("text/plain")) + Expect(body.Parts).To(HaveLen(3)) + Expect(body.Parts[0]).To(Equal(UploadPartInfo{Etag: "fakeetag_1", PartNumber: 1})) + Expect(body.Parts[1]).To(Equal(UploadPartInfo{Etag: "fakeetag_2", PartNumber: 2})) + Expect(body.Parts[2]).To(Equal(UploadPartInfo{Etag: "fakeetag_3", PartNumber: 3})) + Expect(body.Metadata["data-a"]).To(Equal("value-1")) + Expect(body.Metadata["data-b"]).To(Equal("value-2")) + err = json.NewEncoder(w).Encode(map[string]any{}) + Expect(err).NotTo(HaveOccurred()) + return + } + path = strings.TrimPrefix(path, "/") + Expect(path).To(Equal(strconv.FormatUint(uint64(n), 10))) + Expect(r.Method).To(Equal(http.MethodPost)) + + var buf bytes.Buffer + contentLength, err := io.Copy(&buf, r.Body) + Expect(err).NotTo(HaveOccurred()) + Expect(contentLength).To(Equal(int64(len(blocks[n])))) + Expect(buf.Bytes()).To(Equal(blocks[n])) + err = json.NewEncoder(w).Encode(map[string]any{"etag": "fakeetag_" + path}) + Expect(err).NotTo(HaveOccurred()) + }) + }, 5) + + writer, err := bucket.NewWriter(ctx, "existed-file", &blob.WriterOptions{ + ContentType: "text/plain", + Metadata: map[string]string{"data-a": "value-1", "data-b": "value-2"}, + }) + Expect(err).NotTo(HaveOccurred()) + defer writer.Close() + + for _, block := range blocks { + n, err := io.Copy(writer, bytes.NewReader(block)) + Expect(err).NotTo(HaveOccurred()) + Expect(n).To(Equal(int64(len(block)))) + } + + err = writer.Close() + Expect(err).NotTo(HaveOccurred()) + }) + }) + + Context("Copy", func() { + It("should copy object", func(ctx context.Context) { + rsServer.SetHandler(func(w http.ResponseWriter, r *http.Request, _ uint32) { + objectNameSrcBase64ed := base64.URLEncoding.EncodeToString([]byte(bucketName + ":src-file")) + objectNameDstBase64ed := base64.URLEncoding.EncodeToString([]byte(bucketName + ":dst-file")) + Expect(r.Method).To(Equal(http.MethodPost)) + Expect(r.URL.Path).To(Equal("/copy/" + objectNameSrcBase64ed + "/" + objectNameDstBase64ed + "/force/true")) + }, 1) + err := bucket.Copy(ctx, "dst-file", "src-file", nil) + Expect(err).NotTo(HaveOccurred()) + }) + }) + + Context("Delte", func() { + It("should delete object", func(ctx context.Context) { + rsServer.SetHandler(func(w http.ResponseWriter, r *http.Request, _ uint32) { + objectNameDstBase64ed := base64.URLEncoding.EncodeToString([]byte(bucketName + ":dst-file")) + Expect(r.Method).To(Equal(http.MethodPost)) + Expect(r.URL.Path).To(Equal("/delete/" + objectNameDstBase64ed)) + }, 1) + + err := bucket.Delete(ctx, "dst-file") + Expect(err).NotTo(HaveOccurred()) + }) + }) +}) diff --git a/kodoblob/mock_server_test.go b/kodoblob/mock_server_test.go new file mode 100644 index 0000000..6fff441 --- /dev/null +++ b/kodoblob/mock_server_test.go @@ -0,0 +1,78 @@ +package kodoblob_test + +import ( + "net/http" + "net/http/httptest" + "net/url" + "sync/atomic" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +type HandlerFunc func(http.ResponseWriter, *http.Request, uint32) + +type mockServer struct { + server *httptest.Server + handler HandlerFunc + mux *http.ServeMux + max uint32 + n uint32 +} + +func newMockServer() *mockServer { + return newMockServerWithHandler(nil, 0) +} + +func newMockServerWithHandler(handler HandlerFunc, max uint32) *mockServer { + ms := new(mockServer) + ms.SetHandler(handler, max) + ms.server = httptest.NewServer(ms.createMockHandler()) + return ms +} + +func newMockServerWithMux(f func(*http.ServeMux), max uint32) *mockServer { + ms := new(mockServer) + ms.WithMux(f, max) + ms.server = httptest.NewServer(ms.createMockHandler()) + return ms +} + +func (ms *mockServer) createMockHandler() http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + defer GinkgoRecover() + n := atomic.AddUint32(&ms.n, 1) - 1 + if ms.handler != nil && n < ms.max { + ms.handler(w, r, n) + } else if ms.mux != nil && n < ms.max { + ms.mux.ServeHTTP(w, r) + } else { + Fail("should not reach here") + } + }) +} + +func (ms *mockServer) Host() string { + u, err := url.Parse(ms.URL()) + Expect(err).NotTo(HaveOccurred()) + return u.Host +} + +func (ms *mockServer) URL() string { + return ms.server.URL +} + +func (ms *mockServer) Close() { + ms.server.Close() +} + +func (ms *mockServer) SetHandler(handler HandlerFunc, max uint32) { + ms.handler = handler + ms.max = max +} + +func (ms *mockServer) WithMux(f func(*http.ServeMux), max uint32) { + ms.mux = http.NewServeMux() + ms.max = max + f(ms.mux) +} diff --git a/kodoblob/utils_test.go b/kodoblob/utils_test.go new file mode 100644 index 0000000..0600543 --- /dev/null +++ b/kodoblob/utils_test.go @@ -0,0 +1,18 @@ +package kodoblob_test + +import ( + "bytes" + "io" + "math/rand" + "time" + + . "github.com/onsi/gomega" +) + +func randData(n int) []byte { + var buf bytes.Buffer + _, err := io.Copy(&buf, io.LimitReader(rand.New(rand.NewSource(time.Now().UnixNano())), int64(n))) + Expect(err).NotTo(HaveOccurred()) + + return buf.Bytes() +}