SAST-4051/SimID_Fix

README.md

@@ -50,11 +50,12 @@ SAST v9.3 or higher.
 
 Run export with:
 ```
-.\cxsast_exporter --user username --pass password --url http://localhost
+.\cxsast_exporter --user username --pass password --url http://localhost [--simIdVersion simIdV]
 ```
 
  * Replace `username` and `password` with user credentials.
  * Replace `http://localhost` with the url to SAST, if necessary.
+ * Replace `simIdV` with similarity ID version needed, optional, default value is 0.
 
 ## Additional Documentation
 

cmd/args.go

@@ -57,11 +57,13 @@ func GetArgs(cmd *cobra.Command, productName string) internal.Args {
 	if args.IsDefaultProjectActiveSince {
 		args.ProjectsActiveSince = projectsActiveSinceDefaultValue
 	}
-
 	args.OutputPath, err = os.Getwd()
 	if err != nil {
 		panic(err)
 	}
-
+	args.SimIdVersion, err = cmd.Flags().GetInt(simIdVersionArg)
+	if err != nil {
+		panic(err)
+	}
 	return args
 }

cmd/root.go

@@ -26,6 +26,7 @@
 	queryMapping            = "query-mapping"
 	queryMappingPathDefault = "https://raw.githubusercontent.com/Checkmarx/sast-to-ast-export/master/data/mapping.json"
 	nestedTeams             = "nested-teams"
+	simIdVersionArg         = "simIdVersion"
 
 	projectsActiveSinceDefaultValue = 180
 	emptyProjectsActiveSince        = 0
@@ -40,6 +41,8 @@
 // productBuild is defined in Makefile and initialized during build
 var productBuild string
 
+var simIdVersion int
+
 // rootCmd represents the base command when called without any subcommands
 var rootCmd = &cobra.Command{
 	Use:   productName,
@@ -120,6 +123,7 @@
 	rootCmd.Flags().Bool(debugArg, false, "activate debug mode")
 	rootCmd.Flags().BoolP(verboseArg, "v", false, "enable verbose logging to console")
 	rootCmd.Flags().Bool(nestedTeams, false, "include original team structure without flattening")
+	rootCmd.Flags().IntVarP(&simIdVersion, simIdVersionArg, "", 0, "Version of the similarity ID calculation (default 0)")
 
 	if err := rootCmd.MarkFlagRequired(userArg); err != nil {
 		panic(err)

go.mod

@@ -10,11 +10,12 @@ require (
 	github.com/rs/zerolog v1.31.0
 	github.com/spf13/cobra v1.8.0
 	github.com/stretchr/testify v1.8.4
-	go.uber.org/mock v0.3.0
+	go.uber.org/mock v0.4.0
 )
 
 require (
 	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/golang/mock v1.6.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect

go.sum

@@ -7,6 +7,8 @@ github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4Nij
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY=
 github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
+github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc=
+github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs=
 github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
 github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=
 github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k=
@@ -35,13 +37,38 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 go.uber.org/mock v0.3.0 h1:3mUxI1No2/60yUYax92Pt8eNOEecx2D3lcXZh2NEZJo=
 go.uber.org/mock v0.3.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc=
+go.uber.org/mock v0.4.0 h1:VcM4ZOtdbR4f6VXfiOpwpVJDL6lCReaZ6mw31wqh7KU=
+go.uber.org/mock v0.4.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=
 golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=

internal/app/metadata/metadata.go

@@ -20,6 +20,7 @@
 	sourceProvider       interfaces.SourceFileRepo
 	methodLineProvider   interfaces.MethodLineRepo
 	tmpDir               string
+	simIdVersion         int
 }
 
 func NewMetadataFactory(
@@ -28,13 +29,15 @@
 	sourceProvider interfaces.SourceFileRepo,
 	methodLineProvider interfaces.MethodLineRepo,
 	tmpDir string,
+	simIdVersion int,
 ) *Factory {
 	return &Factory{
 		astQueryIDProvider,
 		similarityIDProvider,
 		sourceProvider,
 		methodLineProvider,
 		tmpDir,
+		simIdVersion,
 	}
 }
 
@@ -90,7 +93,7 @@
 					result.ResultID, result.PathID,
 					firstSourceFile.LocalName, result.FirstNode.Name, result.FirstNode.Line, result.FirstNode.Column, methodLines[0],
 					lastSourceFile.LocalName, result.LastNode.Name, result.LastNode.Line, result.LastNode.Column, methodLines[len(methodLines)-1],
-					astQueryID,
+					astQueryID, e.simIdVersion,
 				}
 			}
 			close(similarityCalculationJobs)
@@ -105,6 +108,7 @@
 						job.Filename1, job.Name1, job.Line1, job.Column1, job.MethodLine1,
 						job.Filename2, job.Name2, job.Line2, job.Column2, job.MethodLine2,
 						job.QueryID,
+						job.SimIdVersion,
 					)
 					similarityCalculationResults <- SimilarityCalculationResult{
 						ResultID:     job.ResultID,

internal/app/metadata/metadata_test.go

@@ -83,12 +83,12 @@ func TestMetadataFactory_GetMetadataForQueryAndResult(t *testing.T) {
 	similarityIDProviderMock.EXPECT().Calculate(
 		gomock.Any(), metaResult1.FirstNode.Name, metaResult1.FirstNode.Line, metaResult1.FirstNode.Column, metaResult1Data.MethodLines[0],
 		gomock.Any(), metaResult1.LastNode.Name, metaResult1.LastNode.Line, metaResult1.LastNode.Column, metaResult1Data.MethodLines[3],
-		astQueryID,
+		astQueryID, "0",
 	).Return(metaResult1Data.SimilarityID, nil)
 	similarityIDProviderMock.EXPECT().Calculate(
 		gomock.Any(), metaResult2.FirstNode.Name, metaResult2.FirstNode.Line, metaResult2.FirstNode.Column, metaResult2Data.MethodLines[0],
 		gomock.Any(), metaResult2.LastNode.Name, metaResult2.LastNode.Line, metaResult2.LastNode.Column, metaResult2Data.MethodLines[2],
-		astQueryID,
+		astQueryID, "0",
 	).Return(metaResult2Data.SimilarityID, nil)
 	sourceProviderMock := mock_app_source_file.NewMockSourceFileRepo(ctrl)
 	sourceProviderMock.EXPECT().
@@ -117,7 +117,7 @@ func TestMetadataFactory_GetMetadataForQueryAndResult(t *testing.T) {
 	methodLineProvider.EXPECT().
 		GetMethodLinesByPath(scanID, metaQuery.QueryID).
 		Return(methodLinesResult, nil)
-	metadata := NewMetadataFactory(astQueryIDProviderMock, similarityIDProviderMock, sourceProviderMock, methodLineProvider, tmpDir)
+	metadata := NewMetadataFactory(astQueryIDProviderMock, similarityIDProviderMock, sourceProviderMock, methodLineProvider, tmpDir, 0)
 
 	result, err := metadata.GetMetadataRecord(scanID, []*Query{metaQuery})
 	assert.NoError(t, err)

internal/app/metadata/models.go

@@ -50,6 +50,7 @@
 		Filename1, Name1, Line1, Column1, MethodLine1,
 		Filename2, Name2, Line2, Column2, MethodLine2,
 		QueryID string
+		SimIdVersion int
 	}
 
 	SimilarityCalculationResult struct {

internal/integration/similarity/similarity_id_calculator.go

@@ -1,6 +1,7 @@
 package similarity
 
 import (
+	"fmt"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -18,6 +19,7 @@
 		filename1, name1, line1, column1, methodLine1,
 		filename2, name2, line2, column2, methodLine2,
 		queryID string,
+		simIdVersion int,
 	) (string, error)
 }
 
@@ -39,19 +41,21 @@
 	filename1, name1, line1, column1, methodLine1,
 	filename2, name2, line2, column2, methodLine2,
 	queryID string,
+	simIdVersion int,
 ) (string, error) {
 	command := exec.Command( //nolint:gosec
 		e.calculatorCmd,
 		filename1, name1, line1, column1, methodLine1,
 		filename2, name2, line2, column2, methodLine2,
 		queryID,
+		fmt.Sprint(simIdVersion),
 	)
 	out, err := command.Output()
 	if err != nil {
 		return "", errors.Wrapf(
 			err,
-			"failed running command file1=%s name1=%s line1=%s col1=%s method1=%s file2=%s name2=%s line2=%s col2=%s method2=%s query=%s",
-			filename1, name1, line1, column1, methodLine1, filename2, name2, line2, column2, methodLine2, queryID,
+			"failed running command file1=%s name1=%s line1=%s col1=%s method1=%s file2=%s name2=%s line2=%s col2=%s method2=%s query=%s simIdVersion=%d",
+			filename1, name1, line1, column1, methodLine1, filename2, name2, line2, column2, methodLine2, queryID, simIdVersion,
 		)
 	}
 	return strings.TrimSpace(string(out)), nil

internal/models.go

@@ -18,6 +18,7 @@
 	RunTime          time.Time
 	QueryMappingFile string
 	NestedTeams      bool
+	SimIdVersion     int
 }
 
 type ReportJob struct {

internal/process.go

@@ -159,7 +159,7 @@
 		}
 	}()
 
-	metadataSource := metadata.NewMetadataFactory(astQueryProvider, similarityIDCalculator, sourceRepo, methodLineRepo, metadataTempDir)
+	metadataSource := metadata.NewMetadataFactory(astQueryProvider, similarityIDCalculator, sourceRepo, methodLineRepo, metadataTempDir, args.SimIdVersion)
 
 	addErr := addCustomQueryIDs(astQueryProvider, astQueryMappingProvider)
 	if addErr != nil {
@@ -495,8 +495,6 @@
 		Str("scans", fmt.Sprintf("%v", triagedScans)).
 		Msg("last scans by project")
 
-	log.Info().Msgf("%d results found", len(triagedScans))
-
 	// create and fetch report for each scan
 	go produceReports(triagedScans, reportJobs)
 
@@ -585,6 +583,7 @@
 		for _, project := range *projects {
 			// get triaged results
 			triagedResults, triagedResultsErr := client.GetTriagedResultsByScanID(project.LastScanID)
+			log.Info().Msgf("fetching %d triaged results found from projectId %d scanId %d", len(*triagedResults), project.ID, project.LastScanID)
 			if triagedResultsErr != nil {
 				log.Debug().Err(triagedResultsErr).
 					Int("projectID", project.ID).