From 96ca40e724c4d988ccec19f8a47de771e962b8ea Mon Sep 17 00:00:00 2001 From: Bar Belity Date: Tue, 15 Dec 2020 14:00:39 +0200 Subject: [PATCH] Support including build-dependencies when searching by build (#251) --- artifactory/services/utils/aqlquerybuilder.go | 20 ++++- .../services/utils/artifactoryutils.go | 82 ++++++++++++------- artifactory/services/utils/searchutil.go | 65 +++++++++++++-- artifactory/services/utils/specutils.go | 32 ++++---- 4 files changed, 147 insertions(+), 52 deletions(-) diff --git a/artifactory/services/utils/aqlquerybuilder.go b/artifactory/services/utils/aqlquerybuilder.go index a4636a369..4bb381345 100644 --- a/artifactory/services/utils/aqlquerybuilder.go +++ b/artifactory/services/utils/aqlquerybuilder.go @@ -82,7 +82,7 @@ func handleArchiveSearch(triple RepoPathFile, archivePathFilePairs []RepoPathFil return query } -func createAqlBodyForBuild(buildName, buildNumber string) string { +func createAqlBodyForBuildArtifacts(buildName, buildNumber string) string { itemsPart := `{` + `"artifact.module.build.name":"%s",` + @@ -91,8 +91,22 @@ func createAqlBodyForBuild(buildName, buildNumber string) string { return fmt.Sprintf(itemsPart, buildName, buildNumber) } -func createAqlQueryForBuild(buildName, buildNumber, includeQueryPart string) string { - queryBody := createAqlBodyForBuild(buildName, buildNumber) +func createAqlBodyForBuildDependencies(buildName, buildNumber string) string { + itemsPart := + `{` + + `"dependency.module.build.name":"%s",` + + `"dependency.module.build.number":"%s"` + + `}` + return fmt.Sprintf(itemsPart, buildName, buildNumber) +} + +func createAqlQueryForBuild(buildName, buildNumber, includeQueryPart string, artifactsQuery bool) string { + var queryBody string + if artifactsQuery { + queryBody = createAqlBodyForBuildArtifacts(buildName, buildNumber) + } else { + queryBody = createAqlBodyForBuildDependencies(buildName, buildNumber) + } itemsPart := `items.find(%s)%s` return fmt.Sprintf(itemsPart, queryBody, includeQueryPart) } diff --git a/artifactory/services/utils/artifactoryutils.go b/artifactory/services/utils/artifactoryutils.go index 2731c8a20..db3eeb344 100644 --- a/artifactory/services/utils/artifactoryutils.go +++ b/artifactory/services/utils/artifactoryutils.go @@ -242,47 +242,73 @@ func createBodyForLatestBuildRequest(buildName, buildNumber string) (body []byte } func filterAqlSearchResultsByBuild(specFile *ArtifactoryCommonParams, reader *content.ContentReader, flags CommonConf, itemsAlreadyContainProperties bool) (*content.ContentReader, error) { - var aqlSearchErr error + var artifactsAqlSearchErr, dependenciesAqlSearchErr error var readerWithProps *content.ContentReader - var buildArtifactsSha1 map[string]int + buildArtifactsSha1 := make(map[string]int) + buildDependenciesSha1 := make(map[string]int) var wg sync.WaitGroup - // If 'build-number' is missing in spec file, we fetch the laster from artifactory. + wg.Add(2) + // If 'build-number' is missing in spec file, we fetch the latest from artifactory. buildName, buildNumber, err := getBuildNameAndNumberFromBuildIdentifier(specFile.Build, flags) if err != nil { return nil, err } - wg.Add(1) - // Get Sha1 for artifacts by build name and number go func() { - buildArtifactsSha1, aqlSearchErr = fetchBuildArtifactsSha1(buildName, buildNumber, flags) - wg.Done() + // Get Sha1 for artifacts. + defer wg.Done() + if !specFile.ExcludeArtifacts { + buildArtifactsSha1, artifactsAqlSearchErr = fetchBuildArtifactsOrDependenciesSha1(buildName, buildNumber, flags, true) + } }() - if !itemsAlreadyContainProperties { - // Add properties to the previously found artifacts (in case properties haven't already fetched from Artifactory) - readerWithProps, err = searchProps(specFile.Aql.ItemsFind, "build.name", buildName, flags) - if err != nil { - return nil, err - } - defer readerWithProps.Close() - tempReader, err := loadMissingProperties(reader, readerWithProps) - if err != nil { - return nil, err + go func() { + // Get Sha1 for dependencies. + defer wg.Done() + if specFile.IncludeDeps { + buildDependenciesSha1, dependenciesAqlSearchErr = fetchBuildArtifactsOrDependenciesSha1(buildName, buildNumber, flags, false) } - defer tempReader.Close() + }() + + if specFile.ExcludeArtifacts || itemsAlreadyContainProperties { + // No need to add properties to the search results. wg.Wait() - if aqlSearchErr != nil { - return nil, aqlSearchErr + for k, v := range buildDependenciesSha1 { + buildArtifactsSha1[k] = v + } + if artifactsAqlSearchErr != nil { + return nil, artifactsAqlSearchErr + } + if dependenciesAqlSearchErr != nil { + return nil, dependenciesAqlSearchErr } - return filterBuildAqlSearchResults(tempReader, buildArtifactsSha1, buildName, buildNumber) + return filterBuildAqlSearchResults(reader, buildArtifactsSha1, buildName, buildNumber) } + // Add properties to the previously found artifacts. + readerWithProps, err = searchProps(specFile.Aql.ItemsFind, "build.name", buildName, flags) + if err != nil { + return nil, err + } + defer readerWithProps.Close() + tempReader, err := loadMissingProperties(reader, readerWithProps) + if err != nil { + return nil, err + } + defer tempReader.Close() + wg.Wait() - if aqlSearchErr != nil { - return nil, aqlSearchErr + // Merge artifacts and dependencies Sha1 maps. + for k, v := range buildDependenciesSha1 { + buildArtifactsSha1[k] = v + } + if artifactsAqlSearchErr != nil { + return nil, artifactsAqlSearchErr + } + if dependenciesAqlSearchErr != nil { + return nil, dependenciesAqlSearchErr } - return filterBuildAqlSearchResults(reader, buildArtifactsSha1, buildName, buildNumber) + return filterBuildAqlSearchResults(tempReader, buildArtifactsSha1, buildName, buildNumber) } // Load all properties to the sorted result items. Save the new result items to a file. @@ -351,10 +377,10 @@ func updateProps(readerWithProps *content.ContentReader, resultWriter *content.C return nil } -// Run AQL to retrieve all artifacts associated with a specific build. -// Return a map of the artifacts SHA1. -func fetchBuildArtifactsSha1(buildName, buildNumber string, flags CommonConf) (map[string]int, error) { - buildQuery := createAqlQueryForBuild(buildName, buildNumber, buildIncludeQueryPart([]string{"name", "repo", "path", "actual_sha1"})) +// Run AQL to retrieve artifacts or dependencies which are associated with a specific build. +// Return a map of the items' SHA1. +func fetchBuildArtifactsOrDependenciesSha1(buildName, buildNumber string, flags CommonConf, artifacts bool) (map[string]int, error) { + buildQuery := createAqlQueryForBuild(buildName, buildNumber, buildIncludeQueryPart([]string{"name", "repo", "path", "actual_sha1"}), artifacts) reader, err := aqlSearch(buildQuery, flags) if err != nil { return nil, err diff --git a/artifactory/services/utils/searchutil.go b/artifactory/services/utils/searchutil.go index 76b9144c8..b812013f9 100644 --- a/artifactory/services/utils/searchutil.go +++ b/artifactory/services/utils/searchutil.go @@ -10,6 +10,7 @@ import ( "reflect" "strconv" "strings" + "sync" "github.com/jfrog/jfrog-client-go/artifactory/buildinfo" "github.com/jfrog/jfrog-client-go/utils/errorutils" @@ -29,18 +30,70 @@ const ( ) // Use this function when searching by build without pattern or aql. +// Collect build artifacts and build dependencies separately, then merge the results into one reader. +func SearchBySpecWithBuild(specFile *ArtifactoryCommonParams, flags CommonConf) (*content.ContentReader, error) { + buildName, buildNumber, err := getBuildNameAndNumberFromBuildIdentifier(specFile.Build, flags) + if err != nil { + return nil, err + } + var wg sync.WaitGroup + wg.Add(2) + + // Get build artifacts. + var artifactsReader *content.ContentReader + var artErr error + go func() { + defer wg.Done() + if !specFile.ExcludeArtifacts { + artifactsReader, artErr = getBuildArtifactsForBuildSearch(specFile, flags, buildName, buildNumber) + } + }() + + // Get build dependencies. + var dependenciesReader *content.ContentReader + var depErr error + go func() { + defer wg.Done() + if specFile.IncludeDeps { + dependenciesReader, depErr = getBuildDependenciesForBuildSearch(*specFile, flags, buildName, buildNumber) + } + }() + + wg.Wait() + var readers []*content.ContentReader + if artifactsReader != nil { + defer artifactsReader.Close() + readers = append(readers, artifactsReader) + } + if dependenciesReader != nil { + defer dependenciesReader.Close() + readers = append(readers, dependenciesReader) + } + if artErr != nil { + return nil, err + } + if depErr != nil { + return nil, err + } + + // Return merged readers. + return content.MergeReaders(readers, content.DefaultKey) +} + +func getBuildDependenciesForBuildSearch(specFile ArtifactoryCommonParams, flags CommonConf, buildName, buildNumber string) (*content.ContentReader, error) { + specFile.Aql = Aql{ItemsFind: createAqlBodyForBuildDependencies(buildName, buildNumber)} + executionQuery := BuildQueryFromSpecFile(&specFile, ALL) + return aqlSearch(executionQuery, flags) +} + // Search with builds returns many results, some are not part of the build and others may be duplicated of the same artifact. // 1. Save SHA1 values received for build-name. // 2. Remove artifacts that not are present on the sha1 list // 3. If we have more than one artifact with the same sha1: // 3.1 Compare the build-name & build-number among all the artifact with the same sha1. // This will prevent unnecessary search upon all Artifactory: -func SearchBySpecWithBuild(specFile *ArtifactoryCommonParams, flags CommonConf) (*content.ContentReader, error) { - buildName, buildNumber, err := getBuildNameAndNumberFromBuildIdentifier(specFile.Build, flags) - if err != nil { - return nil, err - } - specFile.Aql = Aql{ItemsFind: createAqlBodyForBuild(buildName, buildNumber)} +func getBuildArtifactsForBuildSearch(specFile *ArtifactoryCommonParams, flags CommonConf, buildName, buildNumber string) (*content.ContentReader, error) { + specFile.Aql = Aql{ItemsFind: createAqlBodyForBuildArtifacts(buildName, buildNumber)} executionQuery := BuildQueryFromSpecFile(specFile, ALL) reader, err := aqlSearch(executionQuery, flags) if err != nil { diff --git a/artifactory/services/utils/specutils.go b/artifactory/services/utils/specutils.go index 63fb236fc..876149b7d 100644 --- a/artifactory/services/utils/specutils.go +++ b/artifactory/services/utils/specutils.go @@ -20,21 +20,23 @@ type ArtifactoryCommonParams struct { Aql Aql Pattern string // Deprecated, use Exclusions instead - ExcludePatterns []string - Exclusions []string - Target string - Props string - ExcludeProps string - SortOrder string - SortBy []string - Offset int - Limit int - Build string - Bundle string - Recursive bool - IncludeDirs bool - Regexp bool - ArchiveEntries string + ExcludePatterns []string + Exclusions []string + Target string + Props string + ExcludeProps string + SortOrder string + SortBy []string + Offset int + Limit int + Build string + ExcludeArtifacts bool + IncludeDeps bool + Bundle string + Recursive bool + IncludeDirs bool + Regexp bool + ArchiveEntries string } type FileGetter interface {