Support including build-dependencies when searching by build (#251)

artifactory/services/utils/aqlquerybuilder.go

@@ -82,7 +82,7 @@ func handleArchiveSearch(triple RepoPathFile, archivePathFilePairs []RepoPathFil
 	return query
 }
 
-func createAqlBodyForBuild(buildName, buildNumber string) string {
+func createAqlBodyForBuildArtifacts(buildName, buildNumber string) string {
 	itemsPart :=
 		`{` +
 			`"artifact.module.build.name":"%s",` +
@@ -91,8 +91,22 @@ func createAqlBodyForBuild(buildName, buildNumber string) string {
 	return fmt.Sprintf(itemsPart, buildName, buildNumber)
 }
 
-func createAqlQueryForBuild(buildName, buildNumber, includeQueryPart string) string {
-	queryBody := createAqlBodyForBuild(buildName, buildNumber)
+func createAqlBodyForBuildDependencies(buildName, buildNumber string) string {
+	itemsPart :=
+		`{` +
+			`"dependency.module.build.name":"%s",` +
+			`"dependency.module.build.number":"%s"` +
+			`}`
+	return fmt.Sprintf(itemsPart, buildName, buildNumber)
+}
+
+func createAqlQueryForBuild(buildName, buildNumber, includeQueryPart string, artifactsQuery bool) string {
+	var queryBody string
+	if artifactsQuery {
+		queryBody = createAqlBodyForBuildArtifacts(buildName, buildNumber)
+	} else {
+		queryBody = createAqlBodyForBuildDependencies(buildName, buildNumber)
+	}
 	itemsPart := `items.find(%s)%s`
 	return fmt.Sprintf(itemsPart, queryBody, includeQueryPart)
 }

artifactory/services/utils/artifactoryutils.go

@@ -242,47 +242,73 @@ func createBodyForLatestBuildRequest(buildName, buildNumber string) (body []byte
 }
 
 func filterAqlSearchResultsByBuild(specFile *ArtifactoryCommonParams, reader *content.ContentReader, flags CommonConf, itemsAlreadyContainProperties bool) (*content.ContentReader, error) {
-	var aqlSearchErr error
+	var artifactsAqlSearchErr, dependenciesAqlSearchErr error
 	var readerWithProps *content.ContentReader
-	var buildArtifactsSha1 map[string]int
+	buildArtifactsSha1 := make(map[string]int)
+	buildDependenciesSha1 := make(map[string]int)
 	var wg sync.WaitGroup
-	// If 'build-number' is missing in spec file, we fetch the laster from artifactory.
+	wg.Add(2)
+	// If 'build-number' is missing in spec file, we fetch the latest from artifactory.
 	buildName, buildNumber, err := getBuildNameAndNumberFromBuildIdentifier(specFile.Build, flags)
 	if err != nil {
 		return nil, err
 	}
 
-	wg.Add(1)
-	// Get Sha1 for artifacts by build name and number
 	go func() {
-		buildArtifactsSha1, aqlSearchErr = fetchBuildArtifactsSha1(buildName, buildNumber, flags)
-		wg.Done()
+		// Get Sha1 for artifacts.
+		defer wg.Done()
+		if !specFile.ExcludeArtifacts {
+			buildArtifactsSha1, artifactsAqlSearchErr = fetchBuildArtifactsOrDependenciesSha1(buildName, buildNumber, flags, true)
+		}
 	}()
 
-	if !itemsAlreadyContainProperties {
-		// Add properties to the previously found artifacts (in case properties haven't already fetched from Artifactory)
-		readerWithProps, err = searchProps(specFile.Aql.ItemsFind, "build.name", buildName, flags)
-		if err != nil {
-			return nil, err
-		}
-		defer readerWithProps.Close()
-		tempReader, err := loadMissingProperties(reader, readerWithProps)
-		if err != nil {
-			return nil, err
+	go func() {
+		// Get Sha1 for dependencies.
+		defer wg.Done()
+		if specFile.IncludeDeps {
+			buildDependenciesSha1, dependenciesAqlSearchErr = fetchBuildArtifactsOrDependenciesSha1(buildName, buildNumber, flags, false)
 		}
-		defer tempReader.Close()
+	}()
+
+	if specFile.ExcludeArtifacts || itemsAlreadyContainProperties {
+		// No need to add properties to the search results.
 		wg.Wait()
-		if aqlSearchErr != nil {
-			return nil, aqlSearchErr
+		for k, v := range buildDependenciesSha1 {
+			buildArtifactsSha1[k] = v
+		}
+		if artifactsAqlSearchErr != nil {
+			return nil, artifactsAqlSearchErr
+		}
+		if dependenciesAqlSearchErr != nil {
+			return nil, dependenciesAqlSearchErr
 		}
-		return filterBuildAqlSearchResults(tempReader, buildArtifactsSha1, buildName, buildNumber)
+		return filterBuildAqlSearchResults(reader, buildArtifactsSha1, buildName, buildNumber)
 	}
 
+	// Add properties to the previously found artifacts.
+	readerWithProps, err = searchProps(specFile.Aql.ItemsFind, "build.name", buildName, flags)
+	if err != nil {
+		return nil, err
+	}
+	defer readerWithProps.Close()
+	tempReader, err := loadMissingProperties(reader, readerWithProps)
+	if err != nil {
+		return nil, err
+	}
+	defer tempReader.Close()
+
 	wg.Wait()
-	if aqlSearchErr != nil {
-		return nil, aqlSearchErr
+	// Merge artifacts and dependencies Sha1 maps.
+	for k, v := range buildDependenciesSha1 {
+		buildArtifactsSha1[k] = v
+	}
+	if artifactsAqlSearchErr != nil {
+		return nil, artifactsAqlSearchErr
+	}
+	if dependenciesAqlSearchErr != nil {
+		return nil, dependenciesAqlSearchErr
 	}
-	return filterBuildAqlSearchResults(reader, buildArtifactsSha1, buildName, buildNumber)
+	return filterBuildAqlSearchResults(tempReader, buildArtifactsSha1, buildName, buildNumber)
 }
 
 // Load all properties to the sorted result items. Save the new result items to a file.
@@ -351,10 +377,10 @@ func updateProps(readerWithProps *content.ContentReader, resultWriter *content.C
 	return nil
 }
 
-// Run AQL to retrieve all artifacts associated with a specific build.
-// Return a map of the artifacts SHA1.
-func fetchBuildArtifactsSha1(buildName, buildNumber string, flags CommonConf) (map[string]int, error) {
-	buildQuery := createAqlQueryForBuild(buildName, buildNumber, buildIncludeQueryPart([]string{"name", "repo", "path", "actual_sha1"}))
+// Run AQL to retrieve artifacts or dependencies which are associated with a specific build.
+// Return a map of the items' SHA1.
+func fetchBuildArtifactsOrDependenciesSha1(buildName, buildNumber string, flags CommonConf, artifacts bool) (map[string]int, error) {
+	buildQuery := createAqlQueryForBuild(buildName, buildNumber, buildIncludeQueryPart([]string{"name", "repo", "path", "actual_sha1"}), artifacts)
 	reader, err := aqlSearch(buildQuery, flags)
 	if err != nil {
 		return nil, err

artifactory/services/utils/searchutil.go

@@ -10,6 +10,7 @@ import (
 	"reflect"
 	"strconv"
 	"strings"
+	"sync"
 
 	"github.com/jfrog/jfrog-client-go/artifactory/buildinfo"
 	"github.com/jfrog/jfrog-client-go/utils/errorutils"
@@ -29,18 +30,70 @@ const (
 )
 
 // Use this function when searching by build without pattern or aql.
+// Collect build artifacts and build dependencies separately, then merge the results into one reader.
+func SearchBySpecWithBuild(specFile *ArtifactoryCommonParams, flags CommonConf) (*content.ContentReader, error) {
+	buildName, buildNumber, err := getBuildNameAndNumberFromBuildIdentifier(specFile.Build, flags)
+	if err != nil {
+		return nil, err
+	}
+	var wg sync.WaitGroup
+	wg.Add(2)
+
+	// Get build artifacts.
+	var artifactsReader *content.ContentReader
+	var artErr error
+	go func() {
+		defer wg.Done()
+		if !specFile.ExcludeArtifacts {
+			artifactsReader, artErr = getBuildArtifactsForBuildSearch(specFile, flags, buildName, buildNumber)
+		}
+	}()
+
+	// Get build dependencies.
+	var dependenciesReader *content.ContentReader
+	var depErr error
+	go func() {
+		defer wg.Done()
+		if specFile.IncludeDeps {
+			dependenciesReader, depErr = getBuildDependenciesForBuildSearch(*specFile, flags, buildName, buildNumber)
+		}
+	}()
+
+	wg.Wait()
+	var readers []*content.ContentReader
+	if artifactsReader != nil {
+		defer artifactsReader.Close()
+		readers = append(readers, artifactsReader)
+	}
+	if dependenciesReader != nil {
+		defer dependenciesReader.Close()
+		readers = append(readers, dependenciesReader)
+	}
+	if artErr != nil {
+		return nil, err
+	}
+	if depErr != nil {
+		return nil, err
+	}
+
+	// Return merged readers.
+	return content.MergeReaders(readers, content.DefaultKey)
+}
+
+func getBuildDependenciesForBuildSearch(specFile ArtifactoryCommonParams, flags CommonConf, buildName, buildNumber string) (*content.ContentReader, error) {
+	specFile.Aql = Aql{ItemsFind: createAqlBodyForBuildDependencies(buildName, buildNumber)}
+	executionQuery := BuildQueryFromSpecFile(&specFile, ALL)
+	return aqlSearch(executionQuery, flags)
+}
+
 // Search with builds returns many results, some are not part of the build and others may be duplicated of the same artifact.
 // 1. Save SHA1 values received for build-name.
 // 2. Remove artifacts that not are present on the sha1 list
 // 3. If we have more than one artifact with the same sha1:
 // 	3.1 Compare the build-name & build-number among all the artifact with the same sha1.
 // This will prevent unnecessary search upon all Artifactory:
-func SearchBySpecWithBuild(specFile *ArtifactoryCommonParams, flags CommonConf) (*content.ContentReader, error) {
-	buildName, buildNumber, err := getBuildNameAndNumberFromBuildIdentifier(specFile.Build, flags)
-	if err != nil {
-		return nil, err
-	}
-	specFile.Aql = Aql{ItemsFind: createAqlBodyForBuild(buildName, buildNumber)}
+func getBuildArtifactsForBuildSearch(specFile *ArtifactoryCommonParams, flags CommonConf, buildName, buildNumber string) (*content.ContentReader, error) {
+	specFile.Aql = Aql{ItemsFind: createAqlBodyForBuildArtifacts(buildName, buildNumber)}
 	executionQuery := BuildQueryFromSpecFile(specFile, ALL)
 	reader, err := aqlSearch(executionQuery, flags)
 	if err != nil {

artifactory/services/utils/specutils.go

@@ -20,21 +20,23 @@ type ArtifactoryCommonParams struct {
 	Aql     Aql
 	Pattern string
 	// Deprecated, use Exclusions instead
-	ExcludePatterns []string
-	Exclusions      []string
-	Target          string
-	Props           string
-	ExcludeProps    string
-	SortOrder       string
-	SortBy          []string
-	Offset          int
-	Limit           int
-	Build           string
-	Bundle          string
-	Recursive       bool
-	IncludeDirs     bool
-	Regexp          bool
-	ArchiveEntries  string
+	ExcludePatterns  []string
+	Exclusions       []string
+	Target           string
+	Props            string
+	ExcludeProps     string
+	SortOrder        string
+	SortBy           []string
+	Offset           int
+	Limit            int
+	Build            string
+	ExcludeArtifacts bool
+	IncludeDeps      bool
+	Bundle           string
+	Recursive        bool
+	IncludeDirs      bool
+	Regexp           bool
+	ArchiveEntries   string
 }
 
 type FileGetter interface {