Merge branch 'dev'

README.md

@@ -395,6 +395,12 @@ bi pipenv [pipenv command] [command options]
 
 Note: checksums calculation is not yet supported for pipenv projects.
 
+#### twine
+
+```shell
+bi twine [twine command] [command options]
+```
+
 #### Dotnet
 
 ```shell

build/build.go

@@ -373,6 +373,14 @@ func (b *Build) GetBuildTimestamp() time.Time {
 	return b.buildTimestamp
 }
 
+func (b *Build) AddArtifacts(moduleId string, moduleType entities.ModuleType, artifacts ...entities.Artifact) error {
+	if !b.buildNameAndNumberProvided() {
+		return errors.New("a build name must be provided in order to add artifacts")
+	}
+	partial := &entities.Partial{ModuleId: moduleId, ModuleType: moduleType, Artifacts: artifacts}
+	return b.SavePartialBuildInfo(partial)
+}
+
 type partialModule struct {
 	moduleType   entities.ModuleType
 	artifacts    map[string]entities.Artifact

build/golang.go

@@ -55,11 +55,7 @@ func (gm *GoModule) SetName(name string) {
 }
 
 func (gm *GoModule) AddArtifacts(artifacts ...entities.Artifact) error {
-	if !gm.containingBuild.buildNameAndNumberProvided() {
-		return errors.New("a build name must be provided in order to add artifacts")
-	}
-	partial := &entities.Partial{ModuleId: gm.name, ModuleType: entities.Go, Artifacts: artifacts}
-	return gm.containingBuild.SavePartialBuildInfo(partial)
+	return gm.containingBuild.AddArtifacts(gm.name, entities.Go, artifacts...)
 }
 
 func (gm *GoModule) loadDependencies() ([]entities.Dependency, error) {

build/npm.go

@@ -97,11 +97,7 @@ func (nm *NpmModule) SetCollectBuildInfo(collectBuildInfo bool) {
 }
 
 func (nm *NpmModule) AddArtifacts(artifacts ...entities.Artifact) error {
-	if !nm.containingBuild.buildNameAndNumberProvided() {
-		return errors.New("a build name must be provided in order to add artifacts")
-	}
-	partial := &entities.Partial{ModuleId: nm.name, ModuleType: entities.Npm, Artifacts: artifacts}
-	return nm.containingBuild.SavePartialBuildInfo(partial)
+	return nm.containingBuild.AddArtifacts(nm.name, entities.Npm, artifacts...)
 }
 
 // This function discards the npm command in npmArgs and keeps only the command flags.

build/python.go

@@ -11,7 +11,7 @@ import (
 type PythonModule struct {
 	containingBuild            *Build
 	tool                       pythonutils.PythonTool
-	name                       string
+	id                         string
 	srcPath                    string
 	localDependenciesPath      string
 	updateDepsChecksumInfoFunc func(dependenciesMap map[string]entities.Dependency, srcPath string) error
@@ -37,41 +37,48 @@ func (pm *PythonModule) RunInstallAndCollectDependencies(commandArgs []string) e
 	if err != nil {
 		return fmt.Errorf("failed while attempting to get %s dependencies graph: %s", pm.tool, err.Error())
 	}
-	// Get package-name.
-	packageName, pkgNameErr := pythonutils.GetPackageName(pm.tool, pm.srcPath)
-	if pkgNameErr != nil {
-		pm.containingBuild.logger.Debug("Couldn't retrieve the package name. Reason:", pkgNameErr.Error())
-	}
-	// If module-name was set by the command, don't change it.
-	if pm.name == "" {
-		// If the package name is unknown, set the module name to be the build name.
-		pm.name = packageName
-		if pm.name == "" {
-			pm.name = pm.containingBuild.buildName
-			pm.containingBuild.logger.Debug(fmt.Sprintf("Using build name: %s as module name.", pm.name))
-		}
-	}
+
+	packageId := pm.SetModuleId()
+
 	if pm.updateDepsChecksumInfoFunc != nil {
 		err = pm.updateDepsChecksumInfoFunc(dependenciesMap, pm.srcPath)
 		if err != nil {
 			return err
 		}
 	}
-	pythonutils.UpdateDepsIdsAndRequestedBy(dependenciesMap, dependenciesGraph, topLevelPackagesList, packageName, pm.name)
-	buildInfoModule := entities.Module{Id: pm.name, Type: entities.Python, Dependencies: dependenciesMapToList(dependenciesMap)}
+	pythonutils.UpdateDepsIdsAndRequestedBy(dependenciesMap, dependenciesGraph, topLevelPackagesList, packageId, pm.id)
+	buildInfoModule := entities.Module{Id: pm.id, Type: entities.Python, Dependencies: dependenciesMapToList(dependenciesMap)}
 	buildInfo := &entities.BuildInfo{Modules: []entities.Module{buildInfoModule}}
 
 	return pm.containingBuild.SaveBuildInfo(buildInfo)
 }
 
+// Sets the module ID and returns the package ID (if found).
+func (pm *PythonModule) SetModuleId() (packageId string) {
+	packageId, pkgNameErr := pythonutils.GetPackageName(pm.tool, pm.srcPath)
+	if pkgNameErr != nil {
+		pm.containingBuild.logger.Debug("Couldn't retrieve the package name. Reason:", pkgNameErr.Error())
+	}
+	// If module-name was set by the command, don't change it.
+	if pm.id == "" {
+		// If the package name is unknown, set the module name to be the build name.
+		pm.id = packageId
+		if pm.id == "" {
+			pm.id = pm.containingBuild.buildName
+			pm.containingBuild.logger.Debug(fmt.Sprintf("Using build name: %s as module name.", pm.id))
+		}
+	}
+	return
+}
+
 // Run install command while parsing the logs for downloaded packages.
 // Populates 'downloadedDependencies' with downloaded package-name and its actual downloaded file (wheel/egg/zip...).
 func (pm *PythonModule) InstallWithLogParsing(commandArgs []string) (map[string]entities.Dependency, error) {
 	return pythonutils.InstallWithLogParsing(pm.tool, commandArgs, pm.containingBuild.logger, pm.srcPath)
 }
 
 func (pm *PythonModule) SetName(name string) {
-	pm.name = name
+	pm.id = name
 }
 
 func (pm *PythonModule) SetLocalDependenciesPath(localDependenciesPath string) {
@@ -81,3 +88,27 @@ func (pm *PythonModule) SetLocalDependenciesPath(localDependenciesPath string) {
 func (pm *PythonModule) SetUpdateDepsChecksumInfoFunc(updateDepsChecksumInfoFunc func(dependenciesMap map[string]entities.Dependency, srcPath string) error) {
 	pm.updateDepsChecksumInfoFunc = updateDepsChecksumInfoFunc
 }
+
+func (pm *PythonModule) TwineUploadWithLogParsing(commandArgs []string) ([]entities.Artifact, error) {
+	pm.SetModuleId()
+	artifactsPaths, err := pythonutils.TwineUploadWithLogParsing(commandArgs, pm.srcPath)
+	if err != nil {
+		return nil, err
+	}
+	return pythonutils.CreateArtifactsFromPaths(artifactsPaths)
+}
+
+func (pm *PythonModule) AddArtifacts(artifacts []entities.Artifact) error {
+	return pm.containingBuild.AddArtifacts(pm.id, entities.Python, artifacts...)
+}
+
+func (pm *PythonModule) TwineUploadAndGenerateBuild(commandArgs []string) error {
+	artifacts, err := pm.TwineUploadWithLogParsing(commandArgs)
+	if err != nil {
+		return err
+	}
+
+	buildInfoModule := entities.Module{Id: pm.id, Type: entities.Python, Artifacts: artifacts}
+	buildInfo := &entities.BuildInfo{Modules: []entities.Module{buildInfoModule}}
+	return pm.containingBuild.SaveBuildInfo(buildInfo)
+}

build/yarn.go

@@ -149,11 +149,7 @@ func (ym *YarnModule) SetTraverseDependenciesFunc(traverseDependenciesFunc func(
 }
 
 func (ym *YarnModule) AddArtifacts(artifacts ...entities.Artifact) error {
-	if !ym.containingBuild.buildNameAndNumberProvided() {
-		return errors.New("a build name must be provided in order to add artifacts")
-	}
-	partial := &entities.Partial{ModuleId: ym.name, ModuleType: entities.Npm, Artifacts: artifacts}
-	return ym.containingBuild.SavePartialBuildInfo(partial)
+	return ym.containingBuild.AddArtifacts(ym.name, entities.Npm, artifacts...)
 }
 
 func validateYarnVersion(executablePath, srcPath string) error {

cli/cli.go

@@ -285,6 +285,36 @@ func GetCommands(logger utils.Log) []*clitool.Command {
 				}
 			},
 		},
+		{
+			Name:      "twine",
+			Usage:     "Generate build-info for a twine project",
+			UsageText: "bi twine",
+			Flags:     flags,
+			Action: func(context *clitool.Context) (err error) {
+				service := build.NewBuildInfoService()
+				service.SetLogger(logger)
+				bld, err := service.GetOrCreateBuild("twine-build", "1")
+				if err != nil {
+					return
+				}
+				defer func() {
+					err = errors.Join(err, bld.Clean())
+				}()
+				pythonModule, err := bld.AddPythonModule("", pythonutils.Twine)
+				if err != nil {
+					return
+				}
+				filteredArgs := filterCliFlags(context.Args().Slice(), flags)
+				if filteredArgs[0] == "upload" {
+					if err := pythonModule.TwineUploadAndGenerateBuild(filteredArgs[1:]); err != nil {
+						return err
+					}
+					return printBuild(bld, context.String(formatFlag))
+				} else {
+					return exec.Command("twine", filteredArgs[1:]...).Run()
+				}
+			},
+		},
 	}
 }
 

entities/buildinfo.go

@@ -6,6 +6,7 @@ import (
 	"golang.org/x/exp/maps"
 	"golang.org/x/exp/slices"
 	"regexp"
+	"sort"
 	"strings"
 	"time"
 
@@ -200,13 +201,24 @@ func (targetBuildInfo *BuildInfo) ToCycloneDxBom() (*cdx.BOM, error) {
 		}
 	}
 
+	sort.Slice(components, func(i, j int) bool {
+		return components[i].BOMRef < components[j].BOMRef
+	})
+
 	// Convert the map of dependencies to CycloneDX dependency objects
 	var dependencies []cdx.Dependency
 	for compRef, deps := range depMap {
 		depsSlice := maps.Keys(deps)
+		sort.Slice(depsSlice, func(i, j int) bool {
+			return depsSlice[i] < depsSlice[j]
+		})
 		dependencies = append(dependencies, cdx.Dependency{Ref: compRef, Dependencies: &depsSlice})
 	}
 
+	sort.Slice(dependencies, func(i, j int) bool {
+		return dependencies[i].Ref < dependencies[j].Ref
+	})
+
 	bom := cdx.NewBOM()
 	bom.Components = &components
 	bom.Dependencies = &dependencies

entities/buildinfo_test.go

@@ -2,6 +2,7 @@ package entities
 
 import (
 	"reflect"
+	"sort"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -239,3 +240,36 @@ func TestAppend(t *testing.T) {
 	assert.NoError(t, err)
 	assert.True(t, results)
 }
+
+func TestToCycloneDxBOM(t *testing.T) {
+	dependencyA := Dependency{Id: "dependency-a", Checksum: Checksum{Sha1: "dependency-a-sha"}, RequestedBy: [][]string{{"dependency-c"}}}
+	dependencyB := Dependency{Id: "dependency-b", Checksum: Checksum{Sha1: "dependency-b-sha"}, RequestedBy: [][]string{{"dependency-b"}, {"dependency-c"}}}
+	dependencyC := Dependency{Id: "dependency-c", Checksum: Checksum{Sha1: "dependency-c-sha"}}
+
+	buildInfo := BuildInfo{
+		Modules: []Module{{
+			Id:           "module-id1",
+			Dependencies: []Dependency{dependencyC, dependencyB, dependencyA},
+		}},
+	}
+
+	cdxBom, err := buildInfo.ToCycloneDxBom()
+	assert.NoError(t, err)
+
+	componentsIsSorted := sort.SliceIsSorted(*cdxBom.Components, func(i, j int) bool {
+		return (*cdxBom.Components)[i].BOMRef < (*cdxBom.Components)[j].BOMRef
+	})
+	assert.True(t, componentsIsSorted)
+
+	dependenciesIsSorted := sort.SliceIsSorted(*cdxBom.Dependencies, func(i, j int) bool {
+		return (*cdxBom.Dependencies)[i].Ref < (*cdxBom.Dependencies)[j].Ref
+	})
+	assert.True(t, dependenciesIsSorted)
+
+	for _, dep := range *cdxBom.Dependencies {
+		dependsOnIsSorted := sort.SliceIsSorted(*dep.Dependencies, func(i, j int) bool {
+			return (*dep.Dependencies)[i] < (*dep.Dependencies)[j]
+		})
+		assert.True(t, dependsOnIsSorted)
+	}
+}

utils/pythonutils/piputils.go

@@ -73,20 +73,20 @@ func writeScriptIfNeeded(targetDirPath, scriptName string) error {
 	return nil
 }
 
-func getPackageNameFromSetuppy(srcPath string) (string, error) {
+func getPackageDetailsFromSetuppy(srcPath string) (packageName string, packageVersion string, err error) {
 	filePath, err := getSetupPyFilePath(srcPath)
 	if err != nil || filePath == "" {
 		// Error was returned or setup.py does not exist in directory.
-		return "", err
+		return
 	}
 
 	// Extract package name from setup.py.
-	packageName, err := ExtractPackageNameFromSetupPy(filePath)
+	packageName, packageVersion, err = extractPackageNameFromSetupPy(filePath)
 	if err != nil {
 		// If setup.py egg_info command failed we use build name as module name and continue to pip-install execution
-		return "", errors.New("couldn't determine module-name after running the 'egg_info' command: " + err.Error())
+		return "", "", errors.New("couldn't determine module-name after running the 'egg_info' command: " + err.Error())
 	}
-	return packageName, nil
+	return packageName, packageVersion, nil
 }
 
 // Look for 'setup.py' file in current work dir.
@@ -95,16 +95,16 @@ func getSetupPyFilePath(srcPath string) (string, error) {
 	return getFilePath(srcPath, "setup.py")
 }
 
-// Get the project-name by running 'egg_info' command on setup.py and extracting it from 'PKG-INFO' file.
-func ExtractPackageNameFromSetupPy(setuppyFilePath string) (string, error) {
+// Get the project name and version by running 'egg_info' command on setup.py and extracting it from 'PKG-INFO' file.
+func extractPackageNameFromSetupPy(setuppyFilePath string) (string, string, error) {
 	// Execute egg_info command and return PKG-INFO content.
 	content, err := getEgginfoPkginfoContent(setuppyFilePath)
 	if err != nil {
-		return "", err
+		return "", "", err
 	}
 
 	// Extract project name from file content.
-	return getProjectIdFromFileContent(content)
+	return getProjectNameAndVersionFromFileContent(content)
 }
 
 // Run egg-info command on setup.py. The command generates metadata files.
@@ -182,24 +182,51 @@ func extractPackageNameFromEggBase(eggBase string) ([]byte, error) {
 
 // Get package ID from PKG-INFO file content.
 // If pattern of package name of version not found, return an error.
-func getProjectIdFromFileContent(content []byte) (string, error) {
+func getProjectNameAndVersionFromFileContent(content []byte) (string, string, error) {
 	// Create package-name regexp.
-	packageNameRegexp := regexp.MustCompile(`(?m)^Name:\s(\w[\w-.]+)`)
+	packageNameWithPrefixRegexp := regexp.MustCompile(`(?m)^Name:\s` + packageNameRegexp)
 
 	// Find first nameMatch of packageNameRegexp.
-	nameMatch := packageNameRegexp.FindStringSubmatch(string(content))
+	nameMatch := packageNameWithPrefixRegexp.FindStringSubmatch(string(content))
 	if len(nameMatch) < 2 {
-		return "", errors.New("failed extracting package name from content")
+		return "", "", errors.New("failed extracting package name from content")
 	}
 
 	// Create package-version regexp.
-	packageVersionRegexp := regexp.MustCompile(`(?m)^Version:\s(\w[\w-.]+)`)
+	packageVersionRegexp := regexp.MustCompile(`(?m)^Version:\s` + packageNameRegexp)
 
 	// Find first match of packageNameRegexp.
 	versionMatch := packageVersionRegexp.FindStringSubmatch(string(content))
 	if len(versionMatch) < 2 {
-		return "", errors.New("failed extracting package version from content")
+		return "", "", errors.New("failed extracting package version from content")
 	}
 
-	return nameMatch[1] + ":" + versionMatch[1], nil
+	return nameMatch[1], versionMatch[1], nil
+}
+
+// Try getting the name and version from pyproject.toml or from setup.py, if those exist.
+func getPipProjectNameAndVersion(srcPath string) (projectName string, projectVersion string, err error) {
+	projectName, projectVersion, err = getPipProjectDetailsFromPyProjectToml(srcPath)
+	if err != nil || projectName != "" {
+		return
+	}
+	return getPackageDetailsFromSetuppy(srcPath)
+}
+
+// Returns project ID based on name and version from pyproject.toml or setup.py, if found.
+func getPipProjectId(srcPath string) (string, error) {
+	projectName, projectVersion, err := getPipProjectNameAndVersion(srcPath)
+	if err != nil || projectName == "" {
+		return "", err
+	}
+	return projectName + ":" + projectVersion, nil
+}
+
+// Try getting the name and version from pyproject.toml.
+func getPipProjectDetailsFromPyProjectToml(srcPath string) (projectName string, projectVersion string, err error) {
+	filePath, err := getPyProjectFilePath(srcPath)
+	if err != nil || filePath == "" {
+		return
+	}
+	return extractPipProjectDetailsFromPyProjectToml(filePath)
 }