diff --git a/artifactory/emptymanager.go b/artifactory/emptymanager.go index 464534e34..2060c10f2 100644 --- a/artifactory/emptymanager.go +++ b/artifactory/emptymanager.go @@ -58,6 +58,7 @@ type ArtifactoryServicesManager interface { GetItemProps(relativePath string) (*utils.ItemProperties, error) UploadFilesWithSummary(params ...services.UploadParams) (operationSummary *utils.OperationSummary, err error) UploadFiles(params ...services.UploadParams) (totalUploaded, totalFailed int, err error) + UploadFilesWithFailFast(params ...services.UploadParams) (totalUploaded, totalFailed int, err error) Copy(params ...services.MoveCopyParams) (successCount, failedCount int, err error) Move(params ...services.MoveCopyParams) (successCount, failedCount int, err error) PublishGoProject(params _go.GoParams) (*utils.OperationSummary, error) @@ -266,6 +267,10 @@ func (esm *EmptyArtifactoryServicesManager) UploadFiles(...services.UploadParams panic("Failed: Method is not implemented") } +func (esm *EmptyArtifactoryServicesManager) UploadFilesWithFailFast(...services.UploadParams) (int, int, error) { + panic("Failed: Method is not implemented") +} + func (esm *EmptyArtifactoryServicesManager) UploadFilesWithSummary(...services.UploadParams) (*utils.OperationSummary, error) { panic("Failed: Method is not implemented") } diff --git a/artifactory/manager.go b/artifactory/manager.go index 80c1cb152..10a785bfe 100644 --- a/artifactory/manager.go +++ b/artifactory/manager.go @@ -33,20 +33,7 @@ func NewWithProgress(config config.Config, progress ioutils.ProgressMgr) (Artifa if err != nil { return nil, err } - client, err := jfroghttpclient.JfrogClientBuilder(). - SetCertificatesPath(config.GetCertificatesPath()). - SetInsecureTls(config.IsInsecureTls()). - SetContext(config.GetContext()). - SetDialTimeout(config.GetDialTimeout()). - SetOverallRequestTimeout(config.GetOverallRequestTimeout()). - SetClientCertPath(artDetails.GetClientCertPath()). - SetClientCertKeyPath(artDetails.GetClientCertKeyPath()). - AppendPreRequestInterceptor(artDetails.RunPreRequestFunctions). - SetContext(config.GetContext()). - SetRetries(config.GetHttpRetries()). - SetRetryWaitMilliSecs(config.GetHttpRetryWaitMilliSecs()). - SetHttpClient(config.GetHttpClient()). - Build() + client, err := buildJFrogHttpClient(config, artDetails) if err != nil { return nil, err } @@ -327,12 +314,21 @@ func (sm *ArtifactoryServicesManagerImp) initUploadService() *services.UploadSer } func (sm *ArtifactoryServicesManagerImp) UploadFiles(params ...services.UploadParams) (totalUploaded, totalFailed int, err error) { + return sm.uploadFiles(sm.initUploadService(), params...) +} + +func (sm *ArtifactoryServicesManagerImp) UploadFilesWithFailFast(params ...services.UploadParams) (totalUploaded, totalFailed int, err error) { uploadService := sm.initUploadService() - summary, e := uploadService.UploadFiles(params...) + uploadService.SetFailFast(true) + return sm.uploadFiles(uploadService, params...) +} + +func (sm *ArtifactoryServicesManagerImp) uploadFiles(uploadService *services.UploadService, uploadParams ...services.UploadParams) (totalUploaded, totalFailed int, err error) { + summary, err := uploadService.UploadFiles(uploadParams...) if summary == nil { - return 0, 0, e + return 0, 0, err } - return summary.TotalSucceeded, summary.TotalFailed, e + return summary.TotalSucceeded, summary.TotalFailed, err } func (sm *ArtifactoryServicesManagerImp) UploadFilesWithSummary(params ...services.UploadParams) (operationSummary *utils.OperationSummary, err error) { @@ -610,3 +606,20 @@ func (sm *ArtifactoryServicesManagerImp) ImportReleaseBundle(filePath string) er releaseService := services.NewReleaseService(sm.config.GetServiceDetails(), sm.client) return releaseService.ImportReleaseBundle(filePath) } + +func buildJFrogHttpClient(config config.Config, authDetails auth.ServiceDetails) (*jfroghttpclient.JfrogHttpClient, error) { + return jfroghttpclient.JfrogClientBuilder(). + SetCertificatesPath(config.GetCertificatesPath()). + SetInsecureTls(config.IsInsecureTls()). + SetContext(config.GetContext()). + SetDialTimeout(config.GetDialTimeout()). + SetOverallRequestTimeout(config.GetOverallRequestTimeout()). + SetClientCertPath(authDetails.GetClientCertPath()). + SetClientCertKeyPath(authDetails.GetClientCertKeyPath()). + AppendPreRequestInterceptor(authDetails.RunPreRequestFunctions). + SetContext(config.GetContext()). + SetRetries(config.GetHttpRetries()). + SetRetryWaitMilliSecs(config.GetHttpRetryWaitMilliSecs()). + SetHttpClient(config.GetHttpClient()). + Build() +} diff --git a/artifactory/services/upload.go b/artifactory/services/upload.go index 4173be09a..250b22678 100644 --- a/artifactory/services/upload.go +++ b/artifactory/services/upload.go @@ -46,8 +46,9 @@ type UploadService struct { ArtDetails auth.ServiceDetails MultipartUpload *utils.MultipartUpload DryRun bool - Threads int saveSummary bool + failFast bool + Threads int resultsManager *resultsManager } @@ -77,6 +78,10 @@ func (us *UploadService) SetSaveSummary(saveSummary bool) { us.saveSummary = saveSummary } +func (us *UploadService) SetFailFast(failFast bool) { + us.failFast = failFast +} + func (us *UploadService) getOperationSummary(totalSucceeded, totalFailed int) *utils.OperationSummary { if !us.saveSummary { return &utils.OperationSummary{ @@ -90,7 +95,7 @@ func (us *UploadService) getOperationSummary(totalSucceeded, totalFailed int) *u func (us *UploadService) UploadFiles(uploadParams ...UploadParams) (summary *utils.OperationSummary, err error) { // Uploading threads are using this struct to report upload results. uploadSummary := utils.NewResult(us.Threads) - producerConsumer := parallel.NewRunner(us.Threads, 20000, false) + producerConsumer := parallel.NewRunner(us.Threads, 20000, us.failFast) errorsQueue := clientutils.NewErrorsQueue(1) if us.saveSummary { us.resultsManager, err = newResultManager() diff --git a/artifactory/services/utils/multipartupload.go b/artifactory/services/utils/multipartupload.go index 87d0994b3..a9178b39e 100644 --- a/artifactory/services/utils/multipartupload.go +++ b/artifactory/services/utils/multipartupload.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "github.com/jfrog/gofrog/crypto" + "github.com/jfrog/gofrog/safeconvert" "io" "net/http" "net/url" @@ -171,20 +172,32 @@ func (mu *MultipartUpload) UploadFileConcurrently(localPath, targetPath string, progressReader = progress.SetMergingState(progressReader.GetId(), false) } + unsignedNumRetries, err := safeconvert.IntToUint(mu.client.GetHttpClient().GetRetries()) + if err != nil { + return "", fmt.Errorf("failed to convert number of retries to uint64: %w", err) + } log.Info(logMsgPrefix + "Starting parts merge...") // The total number of attempts is determined by the number of retries + 1 - return mu.completeAndPollForStatus(logMsgPrefix, uint(mu.client.GetHttpClient().GetRetries())+1, sha1, multipartUploadClient, progressReader) + return mu.completeAndPollForStatus(logMsgPrefix, unsignedNumRetries+1, sha1, multipartUploadClient, progressReader) } func (mu *MultipartUpload) uploadPartsConcurrently(logMsgPrefix string, fileSize, chunkSize int64, splitCount int, localPath string, progressReader ioutils.Progress, multipartUploadClient *httputils.HttpClientDetails) (err error) { numberOfParts := calculateNumberOfParts(fileSize, chunkSize) + unsignedNumOfParts, err := safeconvert.Int64ToUint64(numberOfParts) + if err != nil { + return fmt.Errorf("failed to convert number of parts to uint64: %w", err) + } + unsignedNumRetries, err := safeconvert.Int64ToUint64(int64(mu.client.GetHttpClient().GetRetries())) + if err != nil { + return fmt.Errorf("failed to convert number of retries to uint64: %w", err) + } log.Info(fmt.Sprintf("%sSplitting file to %d parts of %s each, using %d working threads for uploading...", logMsgPrefix, numberOfParts, ConvertIntToStorageSizeString(chunkSize), splitCount)) - producerConsumer := parallel.NewRunner(splitCount, uint(numberOfParts), false) + producerConsumer := parallel.NewRunner(splitCount, uint(unsignedNumOfParts), false) wg := new(sync.WaitGroup) wg.Add(int(numberOfParts)) attemptsAllowed := new(atomic.Uint64) - attemptsAllowed.Add(uint64(numberOfParts) * uint64(mu.client.GetHttpClient().GetRetries())) + attemptsAllowed.Add(unsignedNumOfParts * unsignedNumRetries) go func() { for i := 0; i < int(numberOfParts); i++ { if err = mu.produceUploadTask(producerConsumer, logMsgPrefix, localPath, fileSize, numberOfParts, int64(i), chunkSize, progressReader, multipartUploadClient, attemptsAllowed, wg); err != nil { diff --git a/artifactory/services/utils/searchutil.go b/artifactory/services/utils/searchutil.go index 766a267e1..f2df3c60e 100644 --- a/artifactory/services/utils/searchutil.go +++ b/artifactory/services/utils/searchutil.go @@ -190,7 +190,6 @@ func SearchBySpecWithPattern(specFile *CommonParams, flags CommonConf, requiredA // Use this function when running Aql with pattern func SearchBySpecWithAql(specFile *CommonParams, flags CommonConf, requiredArtifactProps RequiredArtifactProps) (reader *content.ContentReader, err error) { - log.Info("Searching for items in Artifactory...") // Execute the search according to provided aql in specFile. var fetchedProps *content.ContentReader query := BuildQueryFromSpecFile(specFile, requiredArtifactProps) diff --git a/go.mod b/go.mod index 10413b528..7c62e00db 100644 --- a/go.mod +++ b/go.mod @@ -10,14 +10,14 @@ require ( github.com/golang-jwt/jwt/v4 v4.5.0 github.com/gookit/color v1.5.4 github.com/jfrog/archiver/v3 v3.6.1 - github.com/jfrog/build-info-go v1.9.35 - github.com/jfrog/gofrog v1.7.5 + github.com/jfrog/build-info-go v1.9.36 + github.com/jfrog/gofrog v1.7.6 github.com/minio/sha256-simd v1.0.1 github.com/stretchr/testify v1.9.0 github.com/xanzy/ssh-agent v0.3.3 - golang.org/x/crypto v0.26.0 - golang.org/x/exp v0.0.0-20240808152545-0cdaa3abc0fa - golang.org/x/term v0.23.0 + golang.org/x/crypto v0.27.0 + golang.org/x/exp v0.0.0-20240904232852-e7e105dedf7e + golang.org/x/term v0.24.0 ) require ( @@ -52,12 +52,12 @@ require ( golang.org/x/mod v0.20.0 // indirect golang.org/x/net v0.28.0 // indirect golang.org/x/sync v0.8.0 // indirect - golang.org/x/sys v0.23.0 // indirect + golang.org/x/sys v0.25.0 // indirect golang.org/x/tools v0.24.0 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) -// replace github.com/jfrog/build-info-go => github.com/jfrog/build-info-go v1.8.9-0.20240804091815-7407ceb49077 +// replace github.com/jfrog/build-info-go => github.com/jfrog/build-info-go v1.8.9-0.20240909072259-13bf8722d051 -// replace github.com/jfrog/gofrog => github.com/jfrog/gofrog dev +// replace github.com/jfrog/gofrog => github.com/jfrog/gofrog v1.7.6-0.20240909061051-2d36ae4bd05a diff --git a/go.sum b/go.sum index 69ed00e8e..63fbaf494 100644 --- a/go.sum +++ b/go.sum @@ -59,10 +59,10 @@ github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOl github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= github.com/jfrog/archiver/v3 v3.6.1 h1:LOxnkw9pOn45DzCbZNFV6K0+6dCsQ0L8mR3ZcujO5eI= github.com/jfrog/archiver/v3 v3.6.1/go.mod h1:VgR+3WZS4N+i9FaDwLZbq+jeU4B4zctXL+gL4EMzfLw= -github.com/jfrog/build-info-go v1.9.35 h1:P53Ckbuin0GYrq0LWMY0GZSptJcQwiUyW6lqTbXKdcc= -github.com/jfrog/build-info-go v1.9.35/go.mod h1:6mdtqjREK76bHNODXakqKR/+ksJ9dvfLS7H57BZtnLY= -github.com/jfrog/gofrog v1.7.5 h1:dFgtEDefJdlq9cqTRoe09RLxS5Bxbe1Ev5+E6SmZHcg= -github.com/jfrog/gofrog v1.7.5/go.mod h1:jyGiCgiqSSR7k86hcUSu67XVvmvkkgWTmPsH25wI298= +github.com/jfrog/build-info-go v1.9.36 h1:bKoYW3o+U70Zbz2kt5NT84N5JWNxdDXHOf+kVdzK+j4= +github.com/jfrog/build-info-go v1.9.36/go.mod h1:JcISnovFXKx3wWf3p1fcMmlPdt6adxScXvoJN4WXqIE= +github.com/jfrog/gofrog v1.7.6 h1:QmfAiRzVyaI7JYGsB7cxfAJePAZTzFz0gRWZSE27c6s= +github.com/jfrog/gofrog v1.7.6/go.mod h1:ntr1txqNOZtHplmaNd7rS4f8jpA5Apx8em70oYEe7+4= github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= @@ -131,10 +131,10 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= -golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw= -golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54= -golang.org/x/exp v0.0.0-20240808152545-0cdaa3abc0fa h1:ELnwvuAXPNtPk1TJRuGkI9fDTwym6AYBu0qzT8AcHdI= -golang.org/x/exp v0.0.0-20240808152545-0cdaa3abc0fa/go.mod h1:akd2r19cwCdwSwWeIdzYQGa/EZZyqcOdwWiwj5L5eKQ= +golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A= +golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70= +golang.org/x/exp v0.0.0-20240904232852-e7e105dedf7e h1:I88y4caeGeuDQxgdoFPUq097j7kNfw6uvuiNxUBfcBk= +golang.org/x/exp v0.0.0-20240904232852-e7e105dedf7e/go.mod h1:akd2r19cwCdwSwWeIdzYQGa/EZZyqcOdwWiwj5L5eKQ= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.20.0 h1:utOm6MM3R3dnawAiJgn0y+xvuYRsm1RKM/4giyfDgV0= @@ -167,15 +167,15 @@ golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.23.0 h1:YfKFowiIMvtgl1UERQoTPPToxltDeZfbj4H7dVUCwmM= -golang.org/x/sys v0.23.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= +golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= -golang.org/x/term v0.23.0 h1:F6D4vR+EHoL9/sWAWgAR1H2DcHr4PareCbAaCo1RpuU= -golang.org/x/term v0.23.0/go.mod h1:DgV24QBUrK6jhZXl+20l6UWznPlwAHm1Q1mGHtydmSk= +golang.org/x/term v0.24.0 h1:Mh5cbb+Zk2hqqXNO7S1iTjEphVL+jb8ZWaqh/g+JWkM= +golang.org/x/term v0.24.0/go.mod h1:lOBK/LVxemqiMij05LGJ0tzNr8xlmwBRJ81PX6wVLH8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -183,8 +183,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc= -golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= +golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= diff --git a/utils/utils.go b/utils/utils.go index 71232bbd6..fa4f188d9 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -28,7 +28,7 @@ import ( const ( Development = "development" Agent = "jfrog-client-go" - Version = "1.46.1" + Version = "1.46.2" ) type MinVersionProduct string