From 23c20cdcb940e57c1e558fe55be6a2d0757191f2 Mon Sep 17 00:00:00 2001 From: "taowei.wtw" Date: Wed, 1 Apr 2020 10:56:02 +0800 Subject: [PATCH] cp command support set tagging --- lib/command.go | 22 +++++ lib/const.go | 1 + lib/cp.go | 39 ++++++++ lib/cp_test.go | 244 +++++++++++++++++++++++++++++++++++++++++++++++-- lib/option.go | 3 + 5 files changed, 303 insertions(+), 6 deletions(-) diff --git a/lib/command.go b/lib/command.go index 1d354ef1..dcb5c258 100644 --- a/lib/command.go +++ b/lib/command.go @@ -635,6 +635,28 @@ func (cmd *Command) getOSSOptions(hopMap map[string]interface{}, headers map[str return options, nil } +func (cmd *Command) getOSSTagging(strTagging string) ([]oss.Tag, error) { + tags := []oss.Tag{} + strKeys := strings.Split(strTagging, "&") + for _, v := range strKeys { + if v == "" { + return tags, fmt.Errorf("tagging value is empty,maybe exist &&") + } + tagNode := strings.Split(v, "=") + if len(tagNode) >= 3 { + return tags, fmt.Errorf("tagging value error %s", v) + } + + // value maybe empty + tagNode = append(tagNode, "") + tags = append(tags, oss.Tag{ + Key: tagNode[0], + Value: tagNode[1], + }) + } + return tags, nil +} + // GetAllCommands returns all commands list func GetAllCommands() []interface{} { return []interface{}{ diff --git a/lib/const.go b/lib/const.go index 7c3a06a8..11e7d1f8 100644 --- a/lib/const.go +++ b/lib/const.go @@ -80,6 +80,7 @@ const ( OptionRedundancyType = "redundancyType" OptionDisableAllSymlink = "disableAllSymlink" OptionDisableIgnoreError = "disableIgnoreError" + OptionTagging = "tagging" ) // the elements show in stat object diff --git a/lib/cp.go b/lib/cp.go index 676afb9f..0d1c961b 100644 --- a/lib/cp.go +++ b/lib/cp.go @@ -63,6 +63,7 @@ type copyOptionType struct { disableDirObject bool resumeProgress *OssResumeProgressListener disableAllSymlink bool + tagging string } type filterOptionType struct { @@ -250,6 +251,11 @@ var specChineseCopy = SpecText{ 注意：header不区分大小写，但value区分大小写。设置后将用指定的meta代替原来的meta。没有指定的 HTTP HEADER将保留，没有指定的user meta将会被删除。 +--tagging选项 + 该选项在上传文件的同时设置object的tagging信息。当指定--recursive选项时，会设置所有上传的 + objects的tagging信息。 + 如果一次设置多个tagging,必须使用双引号,比如 "tagA=A&tagB=B" + --acl选项 该选项在上传文件的同时设置object的acl信息。当指定--recursive选项时，会设置所有上传的 @@ -527,6 +533,9 @@ var specChineseCopy = SpecText{ ossutil cp local_dir oss://bucket1/b -r --disable-all-symlink 忽略所有的链接子文件以及链接子目录 + ossutil cp local_dir oss://bucket1/b --tagging "tagA=A&tagB=B" -r + 上传的同时设置两个tagging,key分别为tagA和tagB,value分别为A和B + 2) 从oss下载object 假设oss上有下列objects： oss://bucket/abcdir1/a @@ -653,6 +662,9 @@ var specChineseCopy = SpecText{ ossutil cp oss://bucket/dir/ oss://bucket1/ -r --only-current-dir 只copy当前目录下的object, 忽略其他子目录 + + ossutil cp oss://bucket/object1 oss://bucket/object2 --tagging "tagA=A&tagB=B" + copy的同时设置两个tagging,key分别为tagA和tagB,value分别为A和B `, } @@ -777,6 +789,12 @@ var specEnglishCopy = SpecText{ replaced with specified meta. HTTP HEADER will be reserved if no speified value. User meta will be deleted if no specified value. +--tagging option + + This option will set the specified objects' tagging data. If --recursive option is specified, + ossutil will set tagging for all uploaded objects. + If you set more than one tagging at a time, you must use double quotes, such as "tagA=A&tagB=B" + --acl option This option will set acl on the specified objects. If --recursive option is specified, @@ -1083,6 +1101,9 @@ Usage: ossutil cp local_dir oss://bucket1/b -r --disable-all-symlink uploading of symlink files and symlink directories under the local_dir is not allowed + ossutil cp local_dir oss://bucket/b --tagging "tagA=A&tagB=B" + Set two taggings when uploading, the key is tagA and tagB, and the value is A and B + 2) download from oss Suppose there are following objects in oss: oss://bucket/abcdir1/a @@ -1205,6 +1226,9 @@ Usage: ossutil cp oss://bucket/dir/ oss://bucket1/ -r --only-current-dir Copy only the object in the current directory, ignoring other subdirectories + + ossutil cp oss://bucket/object1 oss://bucket/object2 --tagging "tagA=A&tagB=B" + Set two taggings when copying, the key is tagA and tagB, and the value is A and B `, } @@ -1263,6 +1287,7 @@ var copyCommand = CopyCommand{ OptionDisableDirObject, OptionDisableAllSymlink, OptionDisableIgnoreError, + OptionTagging, }, }, } @@ -1299,6 +1324,7 @@ func (cc *CopyCommand) RunCommand() error { cc.cpOption.vrange, _ = GetString(OptionRange, cc.command.options) cc.cpOption.encodingType, _ = GetString(OptionEncodingType, cc.command.options) cc.cpOption.meta, _ = GetString(OptionMeta, cc.command.options) + cc.cpOption.tagging, _ = GetString(OptionTagging, cc.command.options) acl, _ := GetString(OptionACL, cc.command.options) payer, _ := GetString(OptionRequestPayer, cc.command.options) cc.cpOption.partitionInfo, _ = GetString(OptionPartitionDownload, cc.command.options) @@ -1363,6 +1389,18 @@ func (cc *CopyCommand) RunCommand() error { cc.cpOption.options = append(cc.cpOption.options, topts...) } + if cc.cpOption.tagging != "" { + if opType == operationTypeGet { + return fmt.Errorf("No need to set tagging for download") + } + tags, err := cc.command.getOSSTagging(cc.cpOption.tagging) + if err != nil { + return err + } + tagging := oss.Tagging{Tags: tags} + cc.cpOption.options = append(cc.cpOption.options, oss.SetTagging(tagging)) + } + if acl != "" { if opType == operationTypeGet { return fmt.Errorf("No need to set ACL for download") @@ -2931,6 +2969,7 @@ func (cc *CopyCommand) ossCopyObjectRetry(bucket *oss.Bucket, objectName, destBu retryTimes, _ := GetInt(OptionRetryTimes, cc.command.options) options := cc.cpOption.options options = append(options, oss.MetadataDirective(oss.MetaReplace)) + options = append(options, oss.TaggingDirective(oss.TaggingReplace)) for i := 1; ; i++ { if i > 1 { time.Sleep(time.Duration(1) * time.Second) diff --git a/lib/cp_test.go b/lib/cp_test.go index ada8e2b7..5df88afc 100644 --- a/lib/cp_test.go +++ b/lib/cp_test.go @@ -4499,7 +4499,7 @@ func (s *OssutilCommandSuite) TestDownLoadWithoutDisableIgnoreError(c *C) { cpArgs := []string{CloudURLToString(bucketName, ""), dirName} str := "" cpDir := CheckpointDir - routines := 1 + routines := strconv.Itoa(1) recursive := true options := OptionMapType{ "endpoint": &str, @@ -4577,7 +4577,7 @@ func (s *OssutilCommandSuite) TestDownLoadWithDisableIgnoreError(c *C) { cpArgs := []string{CloudURLToString(bucketName, ""), dirName} str := "" cpDir := CheckpointDir - routines := 1 + routines := strconv.Itoa(1) recursive := true disableIgnoreError := true options := OptionMapType{ @@ -4595,7 +4595,7 @@ func (s *OssutilCommandSuite) TestDownLoadWithDisableIgnoreError(c *C) { _, err = cm.RunCommand("cp", cpArgs, options) c.Assert(err, NotNil) - // check,success download 2 file + // check,success download 1 file // exist filePath := dirName + string(os.PathSeparator) + "object1" strObject := s.readFile(filePath, c) @@ -4609,11 +4609,243 @@ func (s *OssutilCommandSuite) TestDownLoadWithDisableIgnoreError(c *C) { // not exist filePath = dirName + string(os.PathSeparator) + "object3" - strObject = s.readFile(filePath, c) - c.Assert(len(strObject) > 0, Equals, true) - os.Remove(filePath) + _, err = os.Stat(filePath) + c.Assert(err, NotNil) os.Remove(testFileName) os.RemoveAll(dirName) s.removeBucket(bucketName, true, c) } + +func (s *OssutilCommandSuite) TestCpWithTaggingError(c *C) { + bucketName := bucketNamePrefix + randLowStr(12) + s.putBucket(bucketName, c) + + // create file + testFileName := "ossutil_test_file" + randStr(5) + data := randStr(100) + s.createFile(testFileName, data, c) + + // upload with tagging + cpArgs := []string{testFileName, CloudURLToString(bucketName, "")} + str := "" + cpDir := CheckpointDir + routines := strconv.Itoa(Routines) + recursive := false + + // invalid format 1 + tagging := "tagA=A&&tagb=B" + options := OptionMapType{ + "endpoint": &str, + "accessKeyID": &str, + "accessKeySecret": &str, + "configFile": &configFile, + "checkpointDir": &cpDir, + "routines": &routines, + "recursive": &recursive, + "tagging": &tagging, + } + + // upload + _, err := cm.RunCommand("cp", cpArgs, options) + c.Assert(err, NotNil) + + // invalid format 2 + tagging = "tagA=A&" + _, err = cm.RunCommand("cp", cpArgs, options) + c.Assert(err, NotNil) + + // invalid format 3 + tagging = "tagA==A" + _, err = cm.RunCommand("cp", cpArgs, options) + c.Assert(err, NotNil) + + // invalid format 4 + tagging = "tagA=A=" + _, err = cm.RunCommand("cp", cpArgs, options) + c.Assert(err, NotNil) + + os.Remove(testFileName) + s.removeBucket(bucketName, true, c) +} + +func (s *OssutilCommandSuite) TestUploadFileWithTagging(c *C) { + bucketName := bucketNamePrefix + randLowStr(12) + s.putBucket(bucketName, c) + + // create file + testFileName := "ossutil_test_file" + randStr(5) + data := randStr(100) + s.createFile(testFileName, data, c) + + // upload with tagging + cpArgs := []string{testFileName, CloudURLToString(bucketName, "")} + str := "" + cpDir := CheckpointDir + routines := strconv.Itoa(Routines) + recursive := false + + tagging := "tagA=A&tagb=B&tagc=C" + options := OptionMapType{ + "endpoint": &str, + "accessKeyID": &str, + "accessKeySecret": &str, + "configFile": &configFile, + "checkpointDir": &cpDir, + "routines": &routines, + "recursive": &recursive, + "tagging": &tagging, + } + + // upload + _, err := cm.RunCommand("cp", cpArgs, options) + c.Assert(err, IsNil) + + objectStat := s.getStat(bucketName, testFileName, c) + c.Assert(objectStat["X-Oss-Tagging-Count"], Equals, "3") + + os.Remove(testFileName) + s.removeBucket(bucketName, true, c) +} + +func (s *OssutilCommandSuite) TestCopyFileWithoutTagging(c *C) { + bucketName := bucketNamePrefix + randLowStr(12) + s.putBucket(bucketName, c) + + // create file + testFileName := "ossutil_test_file" + randStr(5) + data := randStr(100) + s.createFile(testFileName, data, c) + + // upload with tagging + cpArgs := []string{testFileName, CloudURLToString(bucketName, "")} + str := "" + cpDir := CheckpointDir + routines := strconv.Itoa(Routines) + recursive := false + tagging := "tagA=A&tagb=B&tagc=C" + options := OptionMapType{ + "endpoint": &str, + "accessKeyID": &str, + "accessKeySecret": &str, + "configFile": &configFile, + "checkpointDir": &cpDir, + "routines": &routines, + "recursive": &recursive, + "tagging": &tagging, + } + + // upload + _, err := cm.RunCommand("cp", cpArgs, options) + c.Assert(err, IsNil) + objectStat := s.getStat(bucketName, testFileName, c) + c.Assert(objectStat["X-Oss-Tagging-Count"], Equals, "3") + + // then copy file without tagging + destFileName := testFileName + "dest" + srcObjectURL := CloudURLToString(bucketName, testFileName) + destObjectURL := CloudURLToString(bucketName, destFileName) + cpArgs = []string{srcObjectURL, destObjectURL} + delete(options, "tagging") + + _, err = cm.RunCommand("cp", cpArgs, options) + c.Assert(err, IsNil) + + // check dest object, tagging not exist + objectStat = s.getStat(bucketName, destFileName, c) + _, ok := objectStat["X-Oss-Tagging-Count"] + c.Assert(ok, Equals, false) + + os.Remove(testFileName) + s.removeBucket(bucketName, true, c) +} + +func (s *OssutilCommandSuite) TestCopyFileWithTagging(c *C) { + bucketName := bucketNamePrefix + randLowStr(12) + s.putBucket(bucketName, c) + + // create file + testFileName := "ossutil_test_file" + randStr(5) + data := randStr(100) + s.createFile(testFileName, data, c) + + // upload without tagging + cpArgs := []string{testFileName, CloudURLToString(bucketName, "")} + str := "" + cpDir := CheckpointDir + routines := strconv.Itoa(Routines) + recursive := false + + tagging := "tagA=A&tagb=B&tagc=C" + options := OptionMapType{ + "endpoint": &str, + "accessKeyID": &str, + "accessKeySecret": &str, + "configFile": &configFile, + "checkpointDir": &cpDir, + "routines": &routines, + "recursive": &recursive, + } + + // upload + _, err := cm.RunCommand("cp", cpArgs, options) + c.Assert(err, IsNil) + + // then copy file with tagging + destFileName := testFileName + "dest" + srcObjectURL := CloudURLToString(bucketName, testFileName) + destObjectURL := CloudURLToString(bucketName, destFileName) + cpArgs = []string{srcObjectURL, destObjectURL} + + options["tagging"] = &tagging + _, err = cm.RunCommand("cp", cpArgs, options) + c.Assert(err, IsNil) + + // check dest object, tagging exist + objectStat := s.getStat(bucketName, destFileName, c) + c.Assert(objectStat["X-Oss-Tagging-Count"], Equals, "3") + os.Remove(testFileName) + s.removeBucket(bucketName, true, c) +} + +func (s *OssutilCommandSuite) TestUploadMultiFileFileWithTagging(c *C) { + bucketName := bucketNamePrefix + randLowStr(12) + s.putBucket(bucketName, c) + + // create file + fileLen := int64(2 * 1024 * 1024) + testFileName := "ossutil_test_file" + randStr(5) + data := randStr(int(fileLen)) + s.createFile(testFileName, data, c) + + // multipart upload with tagging + cpArgs := []string{testFileName, CloudURLToString(bucketName, "")} + str := "" + cpDir := CheckpointDir + routines := strconv.Itoa(Routines) + recursive := false + threshold := strconv.FormatInt(fileLen/2, 10) + tagging := "tagA=A&tagb=B&tagc=C" + options := OptionMapType{ + "endpoint": &str, + "accessKeyID": &str, + "accessKeySecret": &str, + "configFile": &configFile, + "checkpointDir": &cpDir, + "routines": &routines, + "recursive": &recursive, + "tagging": &tagging, + "bigfileThreshold": &threshold, + } + + // upload + _, err := cm.RunCommand("cp", cpArgs, options) + c.Assert(err, IsNil) + + objectStat := s.getStat(bucketName, testFileName, c) + c.Assert(objectStat["X-Oss-Tagging-Count"], Equals, "3") + c.Assert(objectStat["X-Oss-Object-Type"], Equals, "Multipart") + + os.Remove(testFileName) + s.removeBucket(bucketName, true, c) +} diff --git a/lib/option.go b/lib/option.go index 567c82eb..7e801ffc 100644 --- a/lib/option.go +++ b/lib/option.go @@ -208,6 +208,9 @@ var OptionMap = map[string]Option{ OptionDisableIgnoreError: Option{"", "--disable-ignore-error", "", OptionTypeFlagTrue, "", "", "批量操作时候不忽略错误, 缺省值为false", "specifies that do not ignore errors during batch cp, default value is false"}, + OptionTagging: Option{"", "--tagging", "", OptionTypeString, "", "", + "设置object的tagging,取值格式如[\"TagA=A&TagB=B...\"]", + "Set object tagging, value format is [\"TagA=A&TagB=B...]\""}, } func (T *Option) getHelp(language string) string {