Add twine support

build/build.go

@@ -373,6 +373,14 @@ func (b *Build) GetBuildTimestamp() time.Time {
 	return b.buildTimestamp
 }
 
+func (b *Build) AddArtifacts(moduleId string, moduleType entities.ModuleType, artifacts ...entities.Artifact) error {
+	if !b.buildNameAndNumberProvided() {
+		return errors.New("a build name must be provided in order to add artifacts")
+	}
+	partial := &entities.Partial{ModuleId: moduleId, ModuleType: moduleType, Artifacts: artifacts}
+	return b.SavePartialBuildInfo(partial)
+}
+
 type partialModule struct {
 	moduleType   entities.ModuleType
 	artifacts    map[string]entities.Artifact

build/golang.go

@@ -55,11 +55,7 @@ func (gm *GoModule) SetName(name string) {
 }
 
 func (gm *GoModule) AddArtifacts(artifacts ...entities.Artifact) error {
-	if !gm.containingBuild.buildNameAndNumberProvided() {
-		return errors.New("a build name must be provided in order to add artifacts")
-	}
-	partial := &entities.Partial{ModuleId: gm.name, ModuleType: entities.Go, Artifacts: artifacts}
-	return gm.containingBuild.SavePartialBuildInfo(partial)
+	return gm.containingBuild.AddArtifacts(gm.name, entities.Go, artifacts...)
 }
 
 func (gm *GoModule) loadDependencies() ([]entities.Dependency, error) {

build/npm.go

@@ -97,11 +97,7 @@ func (nm *NpmModule) SetCollectBuildInfo(collectBuildInfo bool) {
 }
 
 func (nm *NpmModule) AddArtifacts(artifacts ...entities.Artifact) error {
-	if !nm.containingBuild.buildNameAndNumberProvided() {
-		return errors.New("a build name must be provided in order to add artifacts")
-	}
-	partial := &entities.Partial{ModuleId: nm.name, ModuleType: entities.Npm, Artifacts: artifacts}
-	return nm.containingBuild.SavePartialBuildInfo(partial)
+	return nm.containingBuild.AddArtifacts(nm.name, entities.Npm, artifacts...)
 }
 
 // This function discards the npm command in npmArgs and keeps only the command flags.

build/python.go

@@ -11,7 +11,7 @@ import (
 type PythonModule struct {
 	containingBuild            *Build
 	tool                       pythonutils.PythonTool
-	name                       string
+	id                         string
 	srcPath                    string
 	localDependenciesPath      string
 	updateDepsChecksumInfoFunc func(dependenciesMap map[string]entities.Dependency, srcPath string) error
@@ -37,41 +37,48 @@ func (pm *PythonModule) RunInstallAndCollectDependencies(commandArgs []string) e
 	if err != nil {
 		return fmt.Errorf("failed while attempting to get %s dependencies graph: %s", pm.tool, err.Error())
 	}
-	// Get package-name.
-	packageName, pkgNameErr := pythonutils.GetPackageName(pm.tool, pm.srcPath)
-	if pkgNameErr != nil {
-		pm.containingBuild.logger.Debug("Couldn't retrieve the package name. Reason:", pkgNameErr.Error())
-	}
-	// If module-name was set by the command, don't change it.
-	if pm.name == "" {
-		// If the package name is unknown, set the module name to be the build name.
-		pm.name = packageName
-		if pm.name == "" {
-			pm.name = pm.containingBuild.buildName
-			pm.containingBuild.logger.Debug(fmt.Sprintf("Using build name: %s as module name.", pm.name))
-		}
-	}
+
+	packageId := pm.SetModuleId()
+
 	if pm.updateDepsChecksumInfoFunc != nil {
 		err = pm.updateDepsChecksumInfoFunc(dependenciesMap, pm.srcPath)
 		if err != nil {
 			return err
 		}
 	}
-	pythonutils.UpdateDepsIdsAndRequestedBy(dependenciesMap, dependenciesGraph, topLevelPackagesList, packageName, pm.name)
-	buildInfoModule := entities.Module{Id: pm.name, Type: entities.Python, Dependencies: dependenciesMapToList(dependenciesMap)}
+	pythonutils.UpdateDepsIdsAndRequestedBy(dependenciesMap, dependenciesGraph, topLevelPackagesList, packageId, pm.id)
+	buildInfoModule := entities.Module{Id: pm.id, Type: entities.Python, Dependencies: dependenciesMapToList(dependenciesMap)}
 	buildInfo := &entities.BuildInfo{Modules: []entities.Module{buildInfoModule}}
 
 	return pm.containingBuild.SaveBuildInfo(buildInfo)
 }
 
+// Sets the module ID and returns the package ID (if found).
+func (pm *PythonModule) SetModuleId() (packageId string) {
+	packageId, pkgNameErr := pythonutils.GetPackageName(pm.tool, pm.srcPath)
+	if pkgNameErr != nil {
+		pm.containingBuild.logger.Debug("Couldn't retrieve the package name. Reason:", pkgNameErr.Error())
+	}
+	// If module-name was set by the command, don't change it.
+	if pm.id == "" {
+		// If the package name is unknown, set the module name to be the build name.
+		pm.id = packageId
+		if pm.id == "" {
+			pm.id = pm.containingBuild.buildName
+			pm.containingBuild.logger.Debug(fmt.Sprintf("Using build name: %s as module name.", pm.id))
+		}
+	}
+	return
+}
+
 // Run install command while parsing the logs for downloaded packages.
 // Populates 'downloadedDependencies' with downloaded package-name and its actual downloaded file (wheel/egg/zip...).
 func (pm *PythonModule) InstallWithLogParsing(commandArgs []string) (map[string]entities.Dependency, error) {
 	return pythonutils.InstallWithLogParsing(pm.tool, commandArgs, pm.containingBuild.logger, pm.srcPath)
 }
 
 func (pm *PythonModule) SetName(name string) {
-	pm.name = name
+	pm.id = name
 }
 
 func (pm *PythonModule) SetLocalDependenciesPath(localDependenciesPath string) {
@@ -81,3 +88,12 @@ func (pm *PythonModule) SetLocalDependenciesPath(localDependenciesPath string) {
 func (pm *PythonModule) SetUpdateDepsChecksumInfoFunc(updateDepsChecksumInfoFunc func(dependenciesMap map[string]entities.Dependency, srcPath string) error) {
 	pm.updateDepsChecksumInfoFunc = updateDepsChecksumInfoFunc
 }
+
+func (pm *PythonModule) TwineUploadWithLogParsing(commandArgs []string) ([]string, error) {
+	pm.SetModuleId()
+	return pythonutils.TwineUploadWithLogParsing(commandArgs, pm.srcPath)
+}
+
+func (pm *PythonModule) AddArtifacts(artifacts []entities.Artifact) error {
+	return pm.containingBuild.AddArtifacts(pm.id, entities.Python, artifacts...)
+}

build/yarn.go

@@ -149,11 +149,7 @@ func (ym *YarnModule) SetTraverseDependenciesFunc(traverseDependenciesFunc func(
 }
 
 func (ym *YarnModule) AddArtifacts(artifacts ...entities.Artifact) error {
-	if !ym.containingBuild.buildNameAndNumberProvided() {
-		return errors.New("a build name must be provided in order to add artifacts")
-	}
-	partial := &entities.Partial{ModuleId: ym.name, ModuleType: entities.Npm, Artifacts: artifacts}
-	return ym.containingBuild.SavePartialBuildInfo(partial)
+	return ym.containingBuild.AddArtifacts(ym.name, entities.Npm, artifacts...)
 }
 
 func validateYarnVersion(executablePath, srcPath string) error {

utils/pythonutils/piputils.go

@@ -73,20 +73,20 @@ func writeScriptIfNeeded(targetDirPath, scriptName string) error {
 	return nil
 }
 
-func getPackageNameFromSetuppy(srcPath string) (string, error) {
+func getPackageDetailsFromSetuppy(srcPath string) (packageName string, packageVersion string, err error) {
 	filePath, err := getSetupPyFilePath(srcPath)
 	if err != nil || filePath == "" {
 		// Error was returned or setup.py does not exist in directory.
-		return "", err
+		return
 	}
 
 	// Extract package name from setup.py.
-	packageName, err := ExtractPackageNameFromSetupPy(filePath)
+	packageName, packageVersion, err = ExtractPackageNameFromSetupPy(filePath)
 	if err != nil {
 		// If setup.py egg_info command failed we use build name as module name and continue to pip-install execution
-		return "", errors.New("couldn't determine module-name after running the 'egg_info' command: " + err.Error())
+		return "", "", errors.New("couldn't determine module-name after running the 'egg_info' command: " + err.Error())
 	}
-	return packageName, nil
+	return packageName, packageVersion, nil
 }
 
 // Look for 'setup.py' file in current work dir.
@@ -95,16 +95,16 @@ func getSetupPyFilePath(srcPath string) (string, error) {
 	return getFilePath(srcPath, "setup.py")
 }
 
-// Get the project-name by running 'egg_info' command on setup.py and extracting it from 'PKG-INFO' file.
-func ExtractPackageNameFromSetupPy(setuppyFilePath string) (string, error) {
+// Get the project name and version by running 'egg_info' command on setup.py and extracting it from 'PKG-INFO' file.
+func ExtractPackageNameFromSetupPy(setuppyFilePath string) (string, string, error) {
 	// Execute egg_info command and return PKG-INFO content.
 	content, err := getEgginfoPkginfoContent(setuppyFilePath)
 	if err != nil {
-		return "", err
+		return "", "", err
 	}
 
 	// Extract project name from file content.
-	return getProjectIdFromFileContent(content)
+	return getProjectNameAndVersionFromFileContent(content)
 }
 
 // Run egg-info command on setup.py. The command generates metadata files.
@@ -182,24 +182,51 @@ func extractPackageNameFromEggBase(eggBase string) ([]byte, error) {
 
 // Get package ID from PKG-INFO file content.
 // If pattern of package name of version not found, return an error.
-func getProjectIdFromFileContent(content []byte) (string, error) {
+func getProjectNameAndVersionFromFileContent(content []byte) (string, string, error) {
 	// Create package-name regexp.
-	packageNameRegexp := regexp.MustCompile(`(?m)^Name:\s(\w[\w-.]+)`)
+	packageNameRegexp := regexp.MustCompile(`(?m)^Name:\s` + _packageNameRegexp)
 
 	// Find first nameMatch of packageNameRegexp.
 	nameMatch := packageNameRegexp.FindStringSubmatch(string(content))
 	if len(nameMatch) < 2 {
-		return "", errors.New("failed extracting package name from content")
+		return "", "", errors.New("failed extracting package name from content")
 	}
 
 	// Create package-version regexp.
-	packageVersionRegexp := regexp.MustCompile(`(?m)^Version:\s(\w[\w-.]+)`)
+	packageVersionRegexp := regexp.MustCompile(`(?m)^Version:\s` + _packageNameRegexp)
 
 	// Find first match of packageNameRegexp.
 	versionMatch := packageVersionRegexp.FindStringSubmatch(string(content))
 	if len(versionMatch) < 2 {
-		return "", errors.New("failed extracting package version from content")
+		return "", "", errors.New("failed extracting package version from content")
 	}
 
-	return nameMatch[1] + ":" + versionMatch[1], nil
+	return nameMatch[1], versionMatch[1], nil
+}
+
+// Try getting the name and version from pyproject.toml or from setup.py, if those exist.
+func GetPipProjectNameAndVersion(srcPath string) (projectName string, projectVersion string, err error) {
+	projectName, projectVersion, err = GetPipProjectDetailsFromPyProjectToml(srcPath)
+	if err != nil || projectName != "" {
+		return
+	}
+	return getPackageDetailsFromSetuppy(srcPath)
+}
+
+// Returns project ID based on name and version from pyproject.toml or setup.py, if found.
+func getPipProjectId(srcPath string) (string, error) {
+	projectName, projectVersion, err := GetPipProjectNameAndVersion(srcPath)
+	if err != nil || projectName == "" {
+		return "", err
+	}
+	return projectName + ":" + projectVersion, nil
+}
+
+// Try getting the name and version from pyproject.toml.
+func GetPipProjectDetailsFromPyProjectToml(srcPath string) (projectName string, projectVersion string, err error) {
+	filePath, err := getPyProjectFilePath(srcPath)
+	if err != nil || filePath == "" {
+		return
+	}
+	return extractPipProjectDetailsFromPyProjectToml(filePath)
 }

utils/pythonutils/piputils_test.go

@@ -9,24 +9,22 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
-func TestGetProjectNameFromFileContent(t *testing.T) {
-	tests := []struct {
-		fileContent         string
-		expectedProjectName string
+func TestGetProjectNameAndVersionFromFileContent(t *testing.T) {
+	testCases := []struct {
+		fileContent            string
+		expectedProjectName    string
+		expectedProjectVersion string
 	}{
-		{"Metadata-Version: 1.0\nName: jfrog-python-example-1\nVersion: 1.0\nSummary: Project example for building Python project with JFrog products\nHome-page: https://github.com/jfrog/project-examples\nAuthor: JFrog\nAuthor-email: [email protected]\nLicense: UNKNOWN\nDescription: UNKNOWN\nPlatform: UNKNOWN", "jfrog-python-example-1:1.0"},
-		{"Metadata-Version: Name: jfrog-python-example-2\nLicense: UNKNOWN\nDescription: UNKNOWN\nPlatform: UNKNOWN\nName: jfrog-python-example-2\nVersion: 1.0\nSummary: Project example for building Python project with JFrog products\nHome-page: https://github.com/jfrog/project-examples\nAuthor: JFrog\nAuthor-email: [email protected]", "jfrog-python-example-2:1.0"},
-		{"Name:Metadata-Version: 3.0\nName: jfrog-python-example-3\nVersion: 1.0\nSummary: Project example for building Python project with JFrog products\nHome-page: https://github.com/jfrog/project-examples\nAuthor: JFrog\nAuthor-email: [email protected]\nName: jfrog-python-example-4", "jfrog-python-example-3:1.0"},
+		{"Metadata-Version: 1.0\nName: jfrog-python-example-1\nVersion: 1.0\nSummary: Project example for building Python project with JFrog products\nHome-page: https://github.com/jfrog/project-examples\nAuthor: JFrog\nAuthor-email: [email protected]\nLicense: UNKNOWN\nDescription: UNKNOWN\nPlatform: UNKNOWN", "jfrog-python-example-1", "1.0"},
+		{"Metadata-Version: Name: jfrog-python-example-2\nLicense: UNKNOWN\nDescription: UNKNOWN\nPlatform: UNKNOWN\nName: jfrog-python-example-2\nVersion: 1.0\nSummary: Project example for building Python project with JFrog products\nHome-page: https://github.com/jfrog/project-examples\nAuthor: JFrog\nAuthor-email: [email protected]", "jfrog-python-example-2", "1.0"},
+		{"Name:Metadata-Version: 3.0\nName: jfrog-python-example-3\nVersion: 1.0\nSummary: Project example for building Python project with JFrog products\nHome-page: https://github.com/jfrog/project-examples\nAuthor: JFrog\nAuthor-email: [email protected]\nName: jfrog-python-example-4", "jfrog-python-example-3", "1.0"},
 	}
 
-	for _, test := range tests {
-		actualValue, err := getProjectIdFromFileContent([]byte(test.fileContent))
-		if err != nil {
-			t.Error(err)
-		}
-		if actualValue != test.expectedProjectName {
-			t.Errorf("Expected value: %s, got: %s.", test.expectedProjectName, actualValue)
-		}
+	for _, test := range testCases {
+		projectName, projectVersion, err := getProjectNameAndVersionFromFileContent([]byte(test.fileContent))
+		assert.NoError(t, err)
+		assert.Equal(t, test.expectedProjectName, projectName)
+		assert.Equal(t, test.expectedProjectVersion, projectVersion)
 	}
 }
 
@@ -40,16 +38,18 @@ var moduleNameTestProvider = []struct {
 	{"setuppyproject", "overidden-module", "overidden-module", "jfrog-python-example:1.0"},
 	{"requirementsproject", "", "", ""},
 	{"requirementsproject", "overidden-module", "overidden-module", ""},
+	{"pyproject", "", "jfrog-python-example:1.0", "pip-project-with-pyproject:1.2.3"},
+	{"pyproject", "overidden-module", "overidden-module", "pip-project-with-pyproject:1.2.3"},
 }
 
-func TestDetermineModuleName(t *testing.T) {
+func TestGetPipProjectId(t *testing.T) {
 	for _, test := range moduleNameTestProvider {
 		t.Run(strings.Join([]string{test.projectName, test.moduleName}, "/"), func(t *testing.T) {
 			tmpProjectPath, cleanup := tests.CreateTestProject(t, filepath.Join("..", "testdata", "pip", test.projectName))
 			defer cleanup()
 
 			// Determine module name
-			packageName, err := getPackageNameFromSetuppy(tmpProjectPath)
+			packageName, err := getPipProjectId(tmpProjectPath)
 			if assert.NoError(t, err) {
 				assert.Equal(t, test.expectedPackageName, packageName)
 			}

utils/pythonutils/poetryutils.go

@@ -9,9 +9,6 @@ import (
 	"golang.org/x/exp/maps"
 )
 
-type PyprojectToml struct {
-	Tool map[string]PoetryPackage
-}
 type PoetryPackage struct {
 	Name            string
 	Version         string
@@ -31,7 +28,7 @@ func getPoetryDependencies(srcPath string) (graph map[string][]string, directDep
 		// Error was returned or poetry.lock does not exist in directory.
 		return map[string][]string{}, []string{}, err
 	}
-	projectName, directDependencies, err := getPackageNameFromPyproject(srcPath)
+	projectName, directDependencies, err := getPoetryPackageFromPyProject(srcPath)
 	if err != nil {
 		return map[string][]string{}, []string{}, err
 	}
@@ -56,49 +53,36 @@ func getPoetryDependencies(srcPath string) (graph map[string][]string, directDep
 	return graph, graph[projectName], nil
 }
 
-func getPackageNameFromPyproject(srcPath string) (string, []string, error) {
-	filePath, err := getPyprojectFilePath(srcPath)
+func getPoetryPackageFromPyProject(srcPath string) (string, []string, error) {
+	filePath, err := getPyProjectFilePath(srcPath)
 	if err != nil || filePath == "" {
-		// Error was returned or pyproject.toml does not exist in directory.
 		return "", []string{}, err
 	}
-	// Extract package name from pyproject.toml.
-	project, err := extractProjectFromPyproject(filePath)
+	project, err := extractPoetryPackageFromPyProjectToml(filePath)
 	if err != nil {
 		return "", []string{}, err
 	}
 	return project.Name, append(maps.Keys(project.Dependencies), maps.Keys(project.DevDependencies)...), nil
 }
 
-// Look for 'pyproject.toml' file in current work dir.
-// If found, return its absolute path.
-func getPyprojectFilePath(srcPath string) (string, error) {
-	return getFilePath(srcPath, "pyproject.toml")
-}
-
 // Look for 'poetry.lock' file in current work dir.
 // If found, return its absolute path.
 func getPoetryLockFilePath(srcPath string) (string, error) {
 	return getFilePath(srcPath, "poetry.lock")
 }
 
-// Get the project-name by parsing the pyproject.toml file.
-func extractProjectFromPyproject(pyprojectFilePath string) (project PoetryPackage, err error) {
-	content, err := os.ReadFile(pyprojectFilePath)
-	if err != nil {
-		return
-	}
-	var pyprojectFile PyprojectToml
-	_, err = toml.Decode(string(content), &pyprojectFile)
+// Get poetry package by parsing the pyproject.toml file.
+func extractPoetryPackageFromPyProjectToml(pyProjectFilePath string) (project PoetryPackage, err error) {
+	pyProjectFile, err := decodePyProjectToml(pyProjectFilePath)
 	if err != nil {
 		return
 	}
-	if poetryProject, ok := pyprojectFile.Tool["poetry"]; ok {
+	if poetryProject, ok := pyProjectFile.Tool["poetry"]; ok {
 		// Extract project name from file content.
 		poetryProject.Name = poetryProject.Name + ":" + poetryProject.Version
 		return poetryProject, nil
 	}
-	return PoetryPackage{}, errors.New("Couldn't find project name and version in " + pyprojectFilePath)
+	return PoetryPackage{}, errors.New("Couldn't find project name and version in " + pyProjectFilePath)
 }
 
 // Get the project-name by parsing the poetry.lock file

utils/pythonutils/poetryutils_test.go

@@ -24,7 +24,7 @@ func TestGetProjectNameFromPyproject(t *testing.T) {
 			tmpProjectPath, cleanup := tests.CreateTestProject(t, filepath.Join("..", "testdata", "poetry", testCase.poetryProject))
 			defer cleanup()
 
-			actualValue, err := extractProjectFromPyproject(filepath.Join(tmpProjectPath, "pyproject.toml"))
+			actualValue, err := extractPoetryPackageFromPyProjectToml(filepath.Join(tmpProjectPath, "pyproject.toml"))
 			assert.NoError(t, err)
 			if actualValue.Name != testCase.expectedProjectName {
 				t.Errorf("Expected value: %s, got: %s.", testCase.expectedProjectName, actualValue)