diff --git a/go.mod b/go.mod index 2fbb3f0..9f0fa49 100644 --- a/go.mod +++ b/go.mod @@ -10,9 +10,9 @@ require ( github.com/cockroachdb/errors v1.11.1 github.com/google/wire v0.6.0 github.com/smartystreets/goconvey v1.8.1 - github.com/spark-lence/tiga v0.0.0-20240517061929-e81eba889226 + github.com/spark-lence/tiga v0.0.0-20240523101823-47fd881052e5 github.com/spf13/cobra v1.8.0 - google.golang.org/genproto/googleapis/api v0.0.0-20240520151616-dc85e6b867a5 + google.golang.org/genproto/googleapis/api v0.0.0-20240521202816-d264139d666e google.golang.org/grpc v1.64.0 google.golang.org/protobuf v1.34.1 ) @@ -135,7 +135,7 @@ require ( golang.org/x/sys v0.20.0 // indirect golang.org/x/text v0.15.0 // indirect golang.org/x/tools v0.21.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240520151616-dc85e6b867a5 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240521202816-d264139d666e // indirect gopkg.in/warnings.v0 v0.1.2 // indirect ) diff --git a/go.sum b/go.sum index b7c4927..5135d41 100644 --- a/go.sum +++ b/go.sum @@ -15,6 +15,7 @@ github.com/agiledragon/gomonkey/v2 v2.11.0/go.mod h1:ap1AmDzcVOAz1YpeJ3TCzIgstoa github.com/allegro/bigcache/v3 v3.1.0 h1:H2Vp8VOvxcrB91o86fUSVJFqeuz8kpyyB02eH3bSzwk= github.com/allegro/bigcache/v3 v3.1.0/go.mod h1:aPyh7jEvrog9zAwx5N7+JUQX5dZTSGpxF1LAR4dr35I= github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= +github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7Dml6nw9rQ= github.com/apapsch/go-jsonmerge/v2 v2.0.0/go.mod h1:lvDnEdqiQrp0O42VQGgmlKpxL1AP2+08jFMw88y4klk= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= @@ -114,6 +115,7 @@ github.com/gorilla/sessions v1.2.1 h1:DHd3rPN5lE3Ts3D8rKkQ8x/0kqfeNmBAaiSi+o7Fsg github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8= github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= @@ -154,6 +156,7 @@ github.com/kisielk/sqlstruct v0.0.0-20201105191214-5f3e10d3ab46/go.mod h1:yyMNCy github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU= github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= @@ -191,6 +194,7 @@ github.com/r3labs/sse/v2 v2.10.0 h1:hFEkLLFY4LDifoHdiCN/LlGBAdVJYsANaLqNYa1l/v0= github.com/r3labs/sse/v2 v2.10.0/go.mod h1:Igau6Whc+F17QUgML1fYe1VPZzTV6EMCnYktEmkNJ7I= github.com/redis/go-redis/v9 v9.5.1 h1:H1X4D3yHPaYrkL5X06Wh6xNVM/pX0Ft4RV0vMGvLBh8= github.com/redis/go-redis/v9 v9.5.1/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M= +github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= @@ -218,6 +222,8 @@ github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9yS github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= github.com/spark-lence/tiga v0.0.0-20240517061929-e81eba889226 h1:WKMb1r+0r5lDKrynMlC9v62EajmtLXMIIk6NzrVbcNs= github.com/spark-lence/tiga v0.0.0-20240517061929-e81eba889226/go.mod h1:MSL8X9t+qvpQ4Tq3vVPKncq9RJcCzF2XGEWkCuNhm6Q= +github.com/spark-lence/tiga v0.0.0-20240523101823-47fd881052e5 h1:lbs5Qf0cOLvIu0d1hftGNYWeudonk77XA6FRxa40yN8= +github.com/spark-lence/tiga v0.0.0-20240523101823-47fd881052e5/go.mod h1:MSL8X9t+qvpQ4Tq3vVPKncq9RJcCzF2XGEWkCuNhm6Q= github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0= @@ -386,8 +392,12 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/genproto/googleapis/api v0.0.0-20240520151616-dc85e6b867a5 h1:P8OJ/WCl/Xo4E4zoe4/bifHpSmmKwARqyqE4nW6J2GQ= google.golang.org/genproto/googleapis/api v0.0.0-20240520151616-dc85e6b867a5/go.mod h1:RGnPtTG7r4i8sPlNyDeikXF99hMM+hN6QMm4ooG9g2g= +google.golang.org/genproto/googleapis/api v0.0.0-20240521202816-d264139d666e h1:SkdGTrROJl2jRGT/Fxv5QUf9jtdKCQh4KQJXbXVLAi0= +google.golang.org/genproto/googleapis/api v0.0.0-20240521202816-d264139d666e/go.mod h1:LweJcLbyVij6rCex8YunD8DYR5VDonap/jYl3ZRxcIU= google.golang.org/genproto/googleapis/rpc v0.0.0-20240520151616-dc85e6b867a5 h1:Q2RxlXqh1cgzzUgV261vBO2jI5R/3DD1J2pM0nI4NhU= google.golang.org/genproto/googleapis/rpc v0.0.0-20240520151616-dc85e6b867a5/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240521202816-d264139d666e h1:Elxv5MwEkCI9f5SkoL6afed6NTdxaGoAo39eANBwHL8= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240521202816-d264139d666e/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= google.golang.org/grpc v1.64.0 h1:KH3VH9y/MgNQg1dE7b3XfVK0GsPSIzJwdF617gUSbvY= google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLpxQYg= google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= @@ -397,6 +407,7 @@ gopkg.in/cenkalti/backoff.v1 v1.1.0/go.mod h1:J6Vskwqd+OMVJl8C33mmtxTBs2gyzfv7UD gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= diff --git a/internal/biz/file/file.go b/internal/biz/file/file.go index 8ee0905..8db371e 100644 --- a/internal/biz/file/file.go +++ b/internal/biz/file/file.go @@ -5,6 +5,14 @@ import ( "crypto/sha256" goErr "errors" "fmt" + "io" + "net/http" + "os" + "path/filepath" + "sort" + "strings" + "time" + "github.com/begonia-org/begonia/internal/pkg/config" "github.com/begonia-org/begonia/internal/pkg/errors" gosdk "github.com/begonia-org/go-sdk" @@ -15,14 +23,6 @@ import ( "github.com/go-git/go-git/v5/plumbing/object" "github.com/spark-lence/tiga" "google.golang.org/grpc/codes" - "io" - "log" - "net/http" - "os" - "path/filepath" - "sort" - "strings" - "time" ) type FileRepo interface { @@ -135,8 +135,8 @@ func (f *FileUsecase) commitFile(dir string, filename string, authorId string, a // 空提交处理 if goErr.Is(err, git.ErrEmptyCommit) { headRef, err := repo.Head() - if err != nil { - return "", err + if err != nil || headRef.Hash().IsZero() { + return "", fmt.Errorf("get head ref error:%w or head ref is nil", err) } return headRef.Hash().String(), nil } @@ -163,12 +163,6 @@ func (f *FileUsecase) checkIn(key string) (string, error) { if key == "" || strings.HasPrefix(key, "/") { return "", gosdk.NewError(errors.ErrInvalidFileKey, int32(api.FileSvrStatus_FILE_INVALIDATE_KEY_ERR), codes.InvalidArgument, "invalid_key") } - // if authorId == "" { - // return "", gosdk.NewError(errors.ErrIdentityMissing, int32(user.UserSvrCode_USER_IDENTITY_MISSING_ERR), codes.InvalidArgument, "not_found_identity") - // } - // if !strings.HasPrefix(key, authorId) { - // key = authorId + "/" + key - // } return key, nil } @@ -348,7 +342,7 @@ func (f *FileUsecase) getPersistenceKeyParts(key string) string { } func (f *FileUsecase) getUri(filePath string) (string, error) { uploadRootDir := f.config.GetUploadDir() - log.Printf("uploadRootDir:%s,filePath:%s", uploadRootDir, filePath) + // log.Printf("uploadRootDir:%s,filePath:%s", uploadRootDir, filePath) uri, err := filepath.Rel(uploadRootDir, filePath) if err != nil { return "", gosdk.NewError(err, int32(common.Code_INTERNAL_ERROR), codes.Internal, "get_file_uri") @@ -380,7 +374,7 @@ func (f *FileUsecase) CompleteMultipartUploadFile(ctx context.Context, in *api.C in.Key = filepath.Join(authorId, key) partsDir := f.getPartsDir(in.UploadId) if !pathExists(partsDir) { - err := gosdk.NewError(errors.ErrUploadIdNotFound, int32(api.FileSvrStatus_FILE_NOT_FOUND_UPLOADID_ERR), codes.NotFound, "upload_id_not_found") + err := gosdk.NewError(fmt.Errorf("%s:%s", in.UploadId, errors.ErrUploadIdNotFound.Error()), int32(api.FileSvrStatus_FILE_NOT_FOUND_UPLOADID_ERR), codes.NotFound, "upload_id_not_found") return nil, err } @@ -396,7 +390,7 @@ func (f *FileUsecase) CompleteMultipartUploadFile(ctx context.Context, in *api.C // merge files to uploadDir/key err = f.mergeFiles(files, filePath) if err != nil { - return nil, gosdk.NewError(fmt.Errorf("merge file error"), int32(common.Code_INTERNAL_ERROR), codes.Internal, "merge_files") + return nil, gosdk.NewError(fmt.Errorf("merge file error:%w", err), int32(common.Code_INTERNAL_ERROR), codes.Internal, "merge_files") } // the parts file has been merged, remove the parts dir to uploadDir/parts/key keyParts := f.getPersistenceKeyParts(in.Key) @@ -420,7 +414,6 @@ func (f *FileUsecase) CompleteMultipartUploadFile(ctx context.Context, in *api.C return nil, gosdk.NewError(err, int32(common.Code_INTERNAL_ERROR), codes.Internal, "commit_file") } } - os.RemoveAll(filepath.Join(f.config.GetUploadDir(), in.UploadId)) return &api.CompleteMultipartUploadResponse{ @@ -442,11 +435,9 @@ func (f *FileUsecase) DownloadForRange(ctx context.Context, in *api.DownloadRequ } file, err := f.getReader(in.Key, in.Version) - if err == git.ErrRepositoryNotExists || os.IsNotExist(err) { - return nil, 0, gosdk.NewError(err, int32(common.Code_NOT_FOUND), codes.NotFound, "file_not_found") - } if err != nil { - return nil, 0, gosdk.NewError(err, int32(common.Code_INTERNAL_ERROR), codes.Internal, "open_file") + code, grcpCode := f.checkStatusCode(err) + return nil, 0, gosdk.NewError(err, code, grcpCode, "open_file") } defer file.Close() @@ -454,9 +445,8 @@ func (f *FileUsecase) DownloadForRange(ctx context.Context, in *api.DownloadRequ if end > 0 { buf = make([]byte, end-start+1) } else { - buf = make([]byte, file.Size()-start+1) + buf = make([]byte, file.Size()-start) } - // log.Printf("start:%d,end:%d,bufsize:%d", start, end, len(buf)) _, err = file.ReadAt(buf, start) if err != nil && err != io.EOF { err = gosdk.NewError(err, int32(common.Code_INTERNAL_ERROR), codes.Internal, "read_file") @@ -474,7 +464,8 @@ func (f *FileUsecase) Metadata(ctx context.Context, in *api.FileMetadataRequest, in.Key = key file, err := f.getReader(in.Key, in.Version) if err != nil { - return nil, gosdk.NewError(err, int32(common.Code_INTERNAL_ERROR), codes.Internal, "open_file") + code,grpcCode:=f.checkStatusCode(err) + return nil, gosdk.NewError(err, code, grpcCode, "open_file") } hasher := sha256.New() @@ -528,7 +519,6 @@ func (f *FileUsecase) getReader(key string, version string) (FileReader, error) var err error if version != "" { fileReader, err = NewFileVersionReader(filePath, version) - // log.Printf("version fileReader:%v", err) if err != nil { return nil, err } @@ -547,13 +537,10 @@ func (f *FileUsecase) Version(ctx context.Context, key, authorId string) (string if err != nil { return "", err } - // fileDir := filepath.Join(f.config.GetUploadDir(), in.Key) file, err := f.getReader(key, "latest") - if err == git.ErrRepositoryNotExists { - return "", gosdk.NewError(err, int32(common.Code_NOT_FOUND), codes.NotFound, "file_not_found") - } if err != nil { - return "", gosdk.NewError(err, int32(common.Code_INTERNAL_ERROR), codes.Internal, "open_file") + code, grpcCode := f.checkStatusCode(err) + return "", gosdk.NewError(err, code, grpcCode, "open_file") } defer file.Close() return file.(FileVersionReader).Version(), nil diff --git a/internal/biz/file/file_test.go b/internal/biz/file/file_test.go index 8a7554e..ab10a5d 100644 --- a/internal/biz/file/file_test.go +++ b/internal/biz/file/file_test.go @@ -22,6 +22,7 @@ import ( api "github.com/begonia-org/go-sdk/api/file/v1" "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/object" cfg "github.com/begonia-org/begonia/internal/pkg/config" "github.com/begonia-org/begonia/internal/pkg/errors" @@ -447,6 +448,50 @@ func testDownload(t *testing.T) { } }) } + c.Convey("test download fail", t, func() { + _, err := fileBiz.Download(context.TODO(), &api.DownloadRequest{ + Key: "/" + fileAuthor + "/test/upload.test1", + }, fileAuthor) + c.So(err, c.ShouldNotBeNil) + c.So(err.Error(), c.ShouldContainSubstring, errors.ErrInvalidFileKey.Error()) + + env := "dev" + if begonia.Env != "" { + env = begonia.Env + + } + config := config.ReadConfig(env) + cnf := cfg.NewConfig(config) + filePath := filepath.Join(cnf.GetUploadDir(), fileAuthor, "test", "upload.test2") + t.Logf("filepath:%s", filePath) + reader, err := file.NewFileVersionReader(filePath, "latest") + c.So(err, c.ShouldBeNil) + patch := gomonkey.ApplyMethodReturn(reader, "Reader", nil, fmt.Errorf("reader error")) + defer patch.Reset() + _, err = fileBiz.Download(context.TODO(), &api.DownloadRequest{ + Key: fileAuthor + "/test/upload.test2", + Version: "latest", + }, fileAuthor) + c.So(err, c.ShouldNotBeNil) + c.So(err.Error(), c.ShouldContainSubstring, "reader error") + patch.Reset() + file, err := file.NewFileVersionReader(filepath.Join(cnf.GetUploadDir(), fileAuthor, "test", "upload.test2"), "latest") + c.So(err, c.ShouldBeNil) + ioReader, err := file.Reader() + c.So(err, c.ShouldBeNil) + patch2 := gomonkey.ApplyMethodReturn(ioReader, "Read", 0, fmt.Errorf("error read file")) + defer patch2.Reset() + _, err = fileBiz.Download(context.TODO(), &api.DownloadRequest{ + Key: fileAuthor + "/test/upload.test2", + Version: "latest", + }, fileAuthor) + + c.So(err, c.ShouldNotBeNil) + c.So(err.Error(), c.ShouldContainSubstring, "error read file") + patch2.Reset() + + // patch2:=gomonkey.ApplyFuncReturn((*object.File).) + }) } func testInitiateUploadFile(t *testing.T) { @@ -477,6 +522,14 @@ func testInitiateUploadFile(t *testing.T) { c.So(err, c.ShouldNotBeNil) c.So(err.Error(), c.ShouldContainSubstring, errors.ErrInvalidFileKey.Error()) + patch := gomonkey.ApplyFuncReturn(os.MkdirAll, fmt.Errorf("mkdir error")) + defer patch.Reset() + _, err = fileBiz.InitiateUploadFile(context.TODO(), &api.InitiateMultipartUploadRequest{ + Key: "test/upload.parts.test1", + }) + c.So(err, c.ShouldNotBeNil) + c.So(err.Error(), c.ShouldContainSubstring, "mkdir error") + }) } @@ -707,14 +760,105 @@ func testCompleteMultipartUploadFile(t *testing.T) { }) c.Convey("test complete parts file fail", t, func() { + _, err := fileBiz.CompleteMultipartUploadFile(context.TODO(), &api.CompleteMultipartUploadRequest{ - Key: "test/upload.parts.test1", + Key: "test/upload.parts.test2", UploadId: "123455678098", }, fileAuthor) c.So(err, c.ShouldNotBeNil) c.So(err.Error(), c.ShouldContainSubstring, errors.ErrUploadIdNotFound.Error()) + _, err = fileBiz.CompleteMultipartUploadFile(context.TODO(), &api.CompleteMultipartUploadRequest{ + Key: "test/upload.parts.test2", + UploadId: "123455678098", + }, "") + c.So(err, c.ShouldNotBeNil) + c.So(err.Error(), c.ShouldContainSubstring, errors.ErrIdentityMissing.Error()) + + _, err = fileBiz.CompleteMultipartUploadFile(context.TODO(), &api.CompleteMultipartUploadRequest{ + Key: "/test/upload.parts.test2", + UploadId: "123455678098", + }, fileAuthor) + c.So(err, c.ShouldNotBeNil) + c.So(err.Error(), c.ShouldContainSubstring, errors.ErrInvalidFileKey.Error()) + + patch := gomonkey.ApplyFuncReturn(os.Lstat, nil, fmt.Errorf("lstat error")) + defer patch.Reset() + rsp, _ := fileBiz.InitiateUploadFile(context.TODO(), &api.InitiateMultipartUploadRequest{ + Key: "test/upload.parts.test2", + }) + uploadId2 := rsp.UploadId + _, err = fileBiz.CompleteMultipartUploadFile(context.TODO(), &api.CompleteMultipartUploadRequest{ + Key: "test/upload.parts.test2", + UploadId: uploadId2, + UseVersion: true, + }, fileAuthor) + c.So(err, c.ShouldNotBeNil) + c.So(err.Error(), c.ShouldContainSubstring, "lstat error") + patch.Reset() + + cases := []struct { + patch interface{} + output []interface{} + err error + }{ + { + patch: os.MkdirAll, + output: []interface{}{fmt.Errorf("mkdir error")}, + err: fmt.Errorf("mkdir error"), + }, { + patch: os.Create, + output: []interface{}{nil, fmt.Errorf("create error")}, + err: fmt.Errorf("create error"), + }, { + patch: os.Open, + output: []interface{}{nil, fmt.Errorf("open error")}, + err: fmt.Errorf("open error"), + }, + { + patch: io.Copy, + output: []interface{}{int64(0), fmt.Errorf("copy error")}, + err: fmt.Errorf("copy error"), + }, + { + patch: os.RemoveAll, + output: []interface{}{fmt.Errorf("remove all error")}, + err: fmt.Errorf("remove all error"), + }, { + patch: filepath.Rel, + output: []interface{}{"", fmt.Errorf("rel error")}, + err: fmt.Errorf("rel error"), + }, { + patch: git.PlainInit, + output: []interface{}{nil, fmt.Errorf("init error")}, + err: fmt.Errorf("init error"), + }, + } + + for _, cases := range cases { + rsp, _ = fileBiz.InitiateUploadFile(context.TODO(), &api.InitiateMultipartUploadRequest{ + Key: "test/upload.parts.test2", + }) + uploadId3 := rsp.UploadId + bigTmpFile, _ := generateRandomFile(1024 * 1024 * 2) + defer os.Remove(bigTmpFile.path) + _ = uploadParts(bigTmpFile.path, uploadId3, "test/upload.parts.test2", t) + os.Remove(bigTmpFile.path) + patch2 := gomonkey.ApplyFuncReturn(cases.patch, cases.output...) + defer patch2.Reset() + _, err = fileBiz.CompleteMultipartUploadFile(context.TODO(), &api.CompleteMultipartUploadRequest{ + Key: "test/upload.parts.test2", + UploadId: uploadId3, + UseVersion: true, + }, fileAuthor) + patch2.Reset() + + c.So(err, c.ShouldNotBeNil) + c.So(err.Error(), c.ShouldContainSubstring, cases.err.Error()) + + } }) + } func testFileMeta(t *testing.T) { fileBiz := newFileBiz() @@ -727,21 +871,61 @@ func testFileMeta(t *testing.T) { c.So(meta, c.ShouldNotBeNil) c.So(meta.Size, c.ShouldEqual, 1024*1024*12) c.So(meta.Sha256, c.ShouldEqual, bigFileSha256) + + meta, err = fileBiz.Metadata(context.Background(), &api.FileMetadataRequest{ + Key: fileAuthor + "/test/upload.parts.test1", + Version: "latest", + }, fileAuthor) + c.So(err, c.ShouldBeNil) + c.So(meta, c.ShouldNotBeNil) + c.So(meta.Size, c.ShouldEqual, 1024*1024*12) + c.So(meta.Sha256, c.ShouldEqual, bigFileSha256) }) c.Convey("test file metadata fail", t, func() { - // _, err := fileBiz.Metadata(context.Background(), &api.FileMetadataRequest{ - // Key: fileAuthor + "/test/upload.parts.test1", - // Version: "", - // }, "") - // c.So(err, c.ShouldNotBeNil) - // c.So(err.Error(), c.ShouldContainSubstring, errors.ErrIdentityMissing.Error()) - _, err := fileBiz.Metadata(context.Background(), &api.FileMetadataRequest{ Key: "", Version: "", }, fileAuthor) c.So(err, c.ShouldNotBeNil) c.So(err.Error(), c.ShouldContainSubstring, errors.ErrInvalidFileKey.Error()) + env := "dev" + if begonia.Env != "" { + env = begonia.Env + } + conf := config.ReadConfig(env) + cnf := cfg.NewConfig(conf) + filePath := filepath.Join(cnf.GetUploadDir(), filepath.Dir(fileAuthor+"/test/upload.parts.test1")) + filePath = filepath.Join(filePath, filepath.Base(fileAuthor+"/test/upload.parts.test1")) + + patch := gomonkey.ApplyFuncReturn(file.NewFileVersionReader, nil, fmt.Errorf("file NewFileVersionReader error")) + defer patch.Reset() + _, err = fileBiz.Metadata(context.Background(), &api.FileMetadataRequest{ + Key: fileAuthor + "/test/upload.parts.test1", + Version: "latest", + }, fileAuthor) + c.So(err, c.ShouldNotBeNil) + c.So(err.Error(), c.ShouldContainSubstring, "file NewFileVersionReader error") + patch.Reset() + + reader, err := file.NewFileVersionReader(filePath, "latest") + c.So(err, c.ShouldBeNil) + patch2 := gomonkey.ApplyMethodReturn(reader, "Reader", nil, fmt.Errorf("file Reader error")) + defer patch2.Reset() + _, err = fileBiz.Metadata(context.Background(), &api.FileMetadataRequest{ + Key: fileAuthor + "/test/upload.parts.test1", + Version: "latest", + }, fileAuthor) + c.So(err, c.ShouldNotBeNil) + c.So(err.Error(), c.ShouldContainSubstring, "file Reader error") + patch2.Reset() + + patch3 := gomonkey.ApplyFuncReturn(io.Copy, int64(0), fmt.Errorf("io.copy error")) + defer patch3.Reset() + _, err = fileBiz.Metadata(context.Background(), &api.FileMetadataRequest{Key: fileAuthor + "/test/upload.parts.test1", + Version: "latest"}, fileAuthor) + c.So(err, c.ShouldNotBeNil) + c.So(err.Error(), c.ShouldContainSubstring, "io.copy error") + }) } func testVersionReader(t *testing.T) { @@ -765,6 +949,80 @@ func testVersionReader(t *testing.T) { c.So(fileReader, c.ShouldNotBeNil) }) + c.Convey("test version reader fail", t, func() { + fileBiz := newFileBiz() + _, err := fileBiz.Version(context.Background(), "/test/version.test", fileAuthor) + c.So(err, c.ShouldNotBeNil) + c.So(err.Error(), c.ShouldContainSubstring, errors.ErrInvalidFileKey.Error()) + patch := gomonkey.ApplyFuncReturn(file.NewFileVersionReader, nil, fmt.Errorf("file NewFileVersionReader error")) + defer patch.Reset() + _, err = fileBiz.Version(context.Background(), fileAuthor+"/test/upload.parts.test1", fileAuthor) + c.So(err, c.ShouldNotBeNil) + c.So(err.Error(), c.ShouldContainSubstring, "file NewFileVersionReader error") + patch.Reset() + patch2 := gomonkey.ApplyFuncReturn(git.PlainOpen, nil, fmt.Errorf("git PlainOpen error")) + defer patch2.Reset() + _, err = file.NewFileVersionReader(fileAuthor+"/test/upload.parts.test1", "latest") + c.So(err, c.ShouldNotBeNil) + c.So(err.Error(), c.ShouldContainSubstring, "git PlainOpen error") + patch2.Reset() + + patch3 := gomonkey.ApplyFuncReturn((*git.Repository).Head, nil, fmt.Errorf("head error")) + defer patch3.Reset() + root := cnf.GetUploadDir() + path := filepath.Join(root, fileAuthor, "test", "upload.parts.test2") + _, err = file.NewFileVersionReader(path, "latest") + c.So(err, c.ShouldNotBeNil) + c.So(err.Error(), c.ShouldContainSubstring, "head error") + patch3.Reset() + + filePath := filepath.Join(cnf.GetUploadDir(), filepath.Dir(fileAuthor+"/test/upload.parts.test1")) + filePath = filepath.Join(filePath, filepath.Base(fileAuthor+"/test/upload.parts.test1")) + reader, err := file.NewFileVersionReader(filePath, "latest") + c.So(err, c.ShouldBeNil) + cases := []struct { + patch interface{} + output []interface{} + err error + }{ + { + patch: (*git.Repository).CommitObject, + output: []interface{}{nil, fmt.Errorf("commit object error")}, + err: fmt.Errorf("commit object error"), + }, + { + patch: (*object.Commit).Tree, + output: []interface{}{nil, fmt.Errorf("tree error")}, + err: fmt.Errorf("tree error"), + }, + { + patch: (*object.Tree).File, + output: []interface{}{nil, fmt.Errorf("file error")}, + err: fmt.Errorf("file error"), + }, + { + patch: io.CopyN, + output: []interface{}{ + int64(0), fmt.Errorf("copy error"), + }, + err: fmt.Errorf("copy error"), + }, + { + patch: io.ReadFull, + output: []interface{}{0, fmt.Errorf("read full error")}, + err: fmt.Errorf("read full error"), + }, + } + for _, cases := range cases { + patch4 := gomonkey.ApplyFuncReturn(cases.patch, cases.output...) + defer patch4.Reset() + _, err := reader.ReadAt(make([]byte, 1024), 512) + c.So(err, c.ShouldNotBeNil) + c.So(err.Error(), c.ShouldContainSubstring, cases.err.Error()) + patch4.Reset() + } + // _, err = fileBiz.Version(context.Background(), fileAuthor+"/test/upload.parts.test1", fileAuthor) + }) } func testDownloadRange(t *testing.T) { fileBiz := newFileBiz() @@ -790,6 +1048,24 @@ func testDownloadRange(t *testing.T) { } b := shaer.Sum(nil) c.So(hex.EncodeToString(b), c.ShouldEqual, bigFileSha256) + + shaer2 := sha256.New() + data, _, err := fileBiz.DownloadForRange(context.Background(), &api.DownloadRequest{ + Key: fileAuthor + "/test/upload.parts.test1", + Version: "latest", + }, 0, 0, fileAuthor) + c.So(err, c.ShouldBeNil) + shaer2.Write(data) + c.So(hex.EncodeToString(shaer2.Sum(nil)), c.ShouldEqual, bigFileSha256) + + shaer3 := sha256.New() + data, _, err = fileBiz.DownloadForRange(context.Background(), &api.DownloadRequest{ + Key: fileAuthor + "/test/upload.parts.test1", + Version: "", + }, 0, 0, fileAuthor) + c.So(err, c.ShouldBeNil) + shaer3.Write(data) + c.So(hex.EncodeToString(shaer3.Sum(nil)), c.ShouldEqual, bigFileSha256) }) c.Convey("test download range parts file fail", t, func() { _, _, err := fileBiz.DownloadForRange(context.Background(), &api.DownloadRequest{ @@ -804,6 +1080,44 @@ func testDownloadRange(t *testing.T) { }, 0, 1024, fileAuthor) c.So(err, c.ShouldNotBeNil) c.So(err.Error(), c.ShouldContainSubstring, errors.ErrInvalidFileKey.Error()) + + _, _, err = fileBiz.DownloadForRange(context.Background(), &api.DownloadRequest{ + Key: "test/upload.parts.test1", + Version: "latest", + }, 1024, 0, fileAuthor) + c.So(err, c.ShouldNotBeNil) + c.So(err.Error(), c.ShouldContainSubstring, errors.ErrInvalidRange.Error()) + patch := gomonkey.ApplyFuncReturn(file.NewFileVersionReader, nil, git.ErrRepositoryNotExists) + defer patch.Reset() + _, _, err = fileBiz.DownloadForRange(context.Background(), &api.DownloadRequest{ + Key: "test/upload.parts.test1", + Version: "latest", + }, 0, 1024, fileAuthor) + patch.Reset() + c.So(err, c.ShouldNotBeNil) + c.So(err.Error(), c.ShouldContainSubstring, git.ErrRepositoryNotExists.Error()) + patch.Reset() + env := "dev" + if begonia.Env != "" { + env = begonia.Env + } + conf := config.ReadConfig(env) + cnf := cfg.NewConfig(conf) + filePath := filepath.Join(cnf.GetUploadDir(), filepath.Dir(fileAuthor+"/test/upload.parts.test1")) + filePath = filepath.Join(filePath, filepath.Base(fileAuthor+"/test/upload.parts.test1")) + reader, err := file.NewFileVersionReader(filePath, "latest") + c.So(err, c.ShouldBeNil) + patch2 := gomonkey.ApplyMethodReturn(reader, "ReadAt", 0, fmt.Errorf("file readAt error")) + defer patch2.Reset() + + _, _, err = fileBiz.DownloadForRange(context.Background(), &api.DownloadRequest{ + Key: fileAuthor + "/test/upload.parts.test1", + Version: "latest", + }, 0, 1024, fileAuthor) + c.So(err, c.ShouldNotBeNil) + c.So(err.Error(), c.ShouldContainSubstring, "file readAt error") + patch2.Reset() + }) } func testDelete(t *testing.T) { @@ -814,16 +1128,7 @@ func testDelete(t *testing.T) { } config := config.ReadConfig(env) cnf := cfg.NewConfig(config) - c.Convey("test delete file success", t, func() { - rsp, err := fileBiz.Delete(context.Background(), &api.DeleteRequest{Key: fileAuthor + "/test/upload.parts.test1"}, fileAuthor) - c.So(err, c.ShouldBeNil) - c.So(rsp, c.ShouldNotBeNil) - _, err = os.Stat(filepath.Join(cnf.GetUploadDir(), fileAuthor, "test", "upload.parts.test1")) - c.So(err, c.ShouldNotBeNil) - _, err = os.Stat(filepath.Join(cnf.GetUploadDir(), "parts", fileAuthor+"/test/upload.parts.test1")) - c.So(err, c.ShouldNotBeNil) - }) c.Convey("test delete file fail", t, func() { patch := gomonkey.ApplyFuncReturn(file.NewFileReader, nil, fmt.Errorf("file not found")) defer patch.Reset() @@ -836,6 +1141,28 @@ func testDelete(t *testing.T) { c.So(err, c.ShouldNotBeNil) c.So(err.Error(), c.ShouldContainSubstring, "not found") + _, err = fileBiz.Delete(context.Background(), &api.DeleteRequest{Key: "/" + fileAuthor + "/test/upload.parts.deleted2"}, fileAuthor) + c.So(err, c.ShouldNotBeNil) + c.So(err.Error(), c.ShouldContainSubstring, errors.ErrInvalidFileKey.Error()) + + patch2 := gomonkey.ApplyFuncReturn(os.RemoveAll, fmt.Errorf("remove all error")) + defer patch2.Reset() + _, err = fileBiz.Delete(context.Background(), &api.DeleteRequest{Key: fileAuthor + "/test/upload.parts.test1"}, fileAuthor) + patch2.Reset() + + c.So(err, c.ShouldNotBeNil) + c.So(err.Error(), c.ShouldContainSubstring, "remove all error") + + }) + c.Convey("test delete file success", t, func() { + rsp, err := fileBiz.Delete(context.Background(), &api.DeleteRequest{Key: fileAuthor + "/test/upload.parts.test1"}, fileAuthor) + c.So(err, c.ShouldBeNil) + c.So(rsp, c.ShouldNotBeNil) + _, err = os.Stat(filepath.Join(cnf.GetUploadDir(), fileAuthor, "test", "upload.parts.test1")) + c.So(err, c.ShouldNotBeNil) + _, err = os.Stat(filepath.Join(cnf.GetUploadDir(), "parts", fileAuthor+"/test/upload.parts.test1")) + c.So(err, c.ShouldNotBeNil) + }) } diff --git a/internal/biz/file/reader.go b/internal/biz/file/reader.go index c2e43e6..4da03fb 100644 --- a/internal/biz/file/reader.go +++ b/internal/biz/file/reader.go @@ -4,6 +4,7 @@ import ( "fmt" "io" "io/fs" + "log" "os" "path/filepath" @@ -148,7 +149,7 @@ func NewFileVersionReader(path string, version string) (FileVersionReader, error fileName := filepath.Base(path) repo, err := git.PlainOpen(dir) if err != nil { - // log.Printf("failed to open git repository: %v", err) + log.Printf("failed to open git repository,%s: %v",dir, err) return nil, err } diff --git a/internal/biz/file/reader_test.go b/internal/biz/file/reader_test.go new file mode 100644 index 0000000..96cbea9 --- /dev/null +++ b/internal/biz/file/reader_test.go @@ -0,0 +1 @@ +package file_test diff --git a/internal/data/data_test.go b/internal/data/data_test.go index 617f0f8..9c7a7e1 100644 --- a/internal/data/data_test.go +++ b/internal/data/data_test.go @@ -1,20 +1,12 @@ package data +import ( + "log" + "testing" +) + // func TestCreateInBatches(t *testing.T) { -// c.Convey("TestCreateInBatches", t, func() { -// env := "dev" -// if begonia.Env != "" { -// env = begonia.Env -// } -// conf := cfg.ReadConfig(env) -// repo := NewDataRepo(conf, gateway.Log) -// repo.db.AutoMigrate(&example.ExampleTable{}) -// snk, _ := tiga.NewSnowflake(1) -// models := []*example.ExampleTable{ -// { -// Uid: snk.GenerateIDString(), -// CreatedAt: tiga.Time(time.Now()), -// }, -// } -// }) -// } +func TestMain(m *testing.M) { + code := m.Run() + log.Printf("All tests passed with code %d", code) +} diff --git a/internal/pkg/config/config.go b/internal/pkg/config/config.go index 68b9446..48e92c0 100644 --- a/internal/pkg/config/config.go +++ b/internal/pkg/config/config.go @@ -186,6 +186,10 @@ func (c *Config) GetRPCPlugins() ([]*goloadbalancer.Server, error) { return plugins, nil } func (c *Config) GetEndpointsPrefix() string { + key:=fmt.Sprintf("%s.etcd.endpoint.prefix",c.GetEnv()) + if val:=c.GetString(key);val!=""{ + return val + } return c.GetString("common.etcd.endpoint.prefix") } @@ -218,6 +222,10 @@ func (c *Config) GetServiceNameKey(name string) string { return filepath.Join(prefix, name) } func (c *Config) GetAppKeyPrefix() string { + key:=fmt.Sprintf("%s.etcd.app.prefix",c.GetEnv()) + if val:=c.GetString(key);val!=""{ + return val + } prefix := c.GetString("common.etcd.app.prefix") return prefix }