diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 633baf1..8c6086d 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -11,15 +11,15 @@ jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2 + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - name: Set up Go - uses: actions/setup-go@bfdd3570ce990073878bf10f6b2d79082de49492 # v2 + uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 with: go-version: 1.21 - name: Run golangci-lint - uses: golangci/golangci-lint-action@5c56cd6c9dc07901af25baab6f2b0d9f3b7c3018 # v2 + uses: golangci/golangci-lint-action@aaa42aa0628b4ae2578232a66b541047968fac86 # v6.1.0 with: version: "v1.54.2" skip-go-installation: true diff --git a/cmd/args.go b/cmd/args.go index cb44e82..76386d0 100644 --- a/cmd/args.go +++ b/cmd/args.go @@ -57,11 +57,13 @@ func GetArgs(cmd *cobra.Command, productName string) internal.Args { if args.IsDefaultProjectActiveSince { args.ProjectsActiveSince = projectsActiveSinceDefaultValue } - args.OutputPath, err = os.Getwd() if err != nil { panic(err) } - + args.SimIDVersion, err = cmd.Flags().GetInt(simIDVersionArg) + if err != nil { + panic(err) + } return args } diff --git a/cmd/root.go b/cmd/root.go index 028b419..f65dff8 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -26,6 +26,7 @@ const ( queryMapping = "query-mapping" queryMappingPathDefault = "https://raw.githubusercontent.com/Checkmarx/sast-to-ast-export/master/data/mapping.json" nestedTeams = "nested-teams" + simIDVersionArg = "simIDVersion" projectsActiveSinceDefaultValue = 180 emptyProjectsActiveSince = 0 @@ -40,6 +41,8 @@ var productVersion string // productBuild is defined in Makefile and initialized during build var productBuild string +var simIDVersion int + // rootCmd represents the base command when called without any subcommands var rootCmd = &cobra.Command{ Use: productName, @@ -56,6 +59,16 @@ Produces: NOTE the minimum supported SAST version is 9.3. SAST installations below this version should be upgraded in order to run this export tool. `, Run: func(cmd *cobra.Command, args []string) { + + // Validate simIDVersion if provided + if simIDVersion < 0 || simIDVersion > 2 { + errorMessage := fmt.Errorf( + "simIDVersion must be 0 (Default), 1 (Trim leading spaces), or 2 (Remove all spaces)", + ) + log.Error().Err(errorMessage).Msg("Invalid simIDVersion") + panic(errorMessage) + } + // setup logging verbose, flagErr := cmd.Flags().GetBool(verboseArg) if flagErr != nil { @@ -120,6 +133,13 @@ func init() { rootCmd.Flags().Bool(debugArg, false, "activate debug mode") rootCmd.Flags().BoolP(verboseArg, "v", false, "enable verbose logging to console") rootCmd.Flags().Bool(nestedTeams, false, "include original team structure without flattening") + rootCmd.Flags().IntVarP( + &simIDVersion, + simIDVersionArg, + "", + 0, + "define version of the similarity ID calculation. Values: 0 - Default, 1 - Trim leading spaces, 2 - Remove all spaces.", + ) if err := rootCmd.MarkFlagRequired(userArg); err != nil { panic(err) diff --git a/external/windows/amd64/SimilarityCalculator.exe b/external/windows/amd64/SimilarityCalculator.exe index f181dcb..06a9c83 100644 Binary files a/external/windows/amd64/SimilarityCalculator.exe and b/external/windows/amd64/SimilarityCalculator.exe differ diff --git a/go.mod b/go.mod index afd5db2..a77f043 100644 --- a/go.mod +++ b/go.mod @@ -10,11 +10,12 @@ require ( github.com/rs/zerolog v1.31.0 github.com/spf13/cobra v1.8.0 github.com/stretchr/testify v1.8.4 - go.uber.org/mock v0.3.0 + go.uber.org/mock v0.4.0 ) require ( github.com/davecgh/go-spew v1.1.1 // indirect + github.com/golang/mock v1.6.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect diff --git a/go.sum b/go.sum index 9a5ca2d..099eee1 100644 --- a/go.sum +++ b/go.sum @@ -7,6 +7,8 @@ github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4Nij github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY= github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= +github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= +github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k= @@ -35,13 +37,38 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= go.uber.org/mock v0.3.0 h1:3mUxI1No2/60yUYax92Pt8eNOEecx2D3lcXZh2NEZJo= go.uber.org/mock v0.3.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc= +go.uber.org/mock v0.4.0 h1:VcM4ZOtdbR4f6VXfiOpwpVJDL6lCReaZ6mw31wqh7KU= +go.uber.org/mock v0.4.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/internal/app/metadata/metadata.go b/internal/app/metadata/metadata.go index 0625b25..d38b8ef 100644 --- a/internal/app/metadata/metadata.go +++ b/internal/app/metadata/metadata.go @@ -20,6 +20,7 @@ type Factory struct { sourceProvider interfaces.SourceFileRepo methodLineProvider interfaces.MethodLineRepo tmpDir string + simIDVersion int } func NewMetadataFactory( @@ -28,6 +29,7 @@ func NewMetadataFactory( sourceProvider interfaces.SourceFileRepo, methodLineProvider interfaces.MethodLineRepo, tmpDir string, + simIDVersion int, ) *Factory { return &Factory{ astQueryIDProvider, @@ -35,6 +37,7 @@ func NewMetadataFactory( sourceProvider, methodLineProvider, tmpDir, + simIDVersion, } } @@ -90,7 +93,7 @@ func (e *Factory) GetMetadataRecord(scanID string, queries []*Query) (*Record, e result.ResultID, result.PathID, firstSourceFile.LocalName, result.FirstNode.Name, result.FirstNode.Line, result.FirstNode.Column, methodLines[0], lastSourceFile.LocalName, result.LastNode.Name, result.LastNode.Line, result.LastNode.Column, methodLines[len(methodLines)-1], - astQueryID, + astQueryID, e.simIDVersion, } } close(similarityCalculationJobs) @@ -105,6 +108,7 @@ func (e *Factory) GetMetadataRecord(scanID string, queries []*Query) (*Record, e job.Filename1, job.Name1, job.Line1, job.Column1, job.MethodLine1, job.Filename2, job.Name2, job.Line2, job.Column2, job.MethodLine2, job.QueryID, + job.SimIDVersion, ) similarityCalculationResults <- SimilarityCalculationResult{ ResultID: job.ResultID, diff --git a/internal/app/metadata/metadata_test.go b/internal/app/metadata/metadata_test.go index e07a887..1bea1be 100644 --- a/internal/app/metadata/metadata_test.go +++ b/internal/app/metadata/metadata_test.go @@ -83,12 +83,12 @@ func TestMetadataFactory_GetMetadataForQueryAndResult(t *testing.T) { similarityIDProviderMock.EXPECT().Calculate( gomock.Any(), metaResult1.FirstNode.Name, metaResult1.FirstNode.Line, metaResult1.FirstNode.Column, metaResult1Data.MethodLines[0], gomock.Any(), metaResult1.LastNode.Name, metaResult1.LastNode.Line, metaResult1.LastNode.Column, metaResult1Data.MethodLines[3], - astQueryID, + astQueryID, 0, ).Return(metaResult1Data.SimilarityID, nil) similarityIDProviderMock.EXPECT().Calculate( gomock.Any(), metaResult2.FirstNode.Name, metaResult2.FirstNode.Line, metaResult2.FirstNode.Column, metaResult2Data.MethodLines[0], gomock.Any(), metaResult2.LastNode.Name, metaResult2.LastNode.Line, metaResult2.LastNode.Column, metaResult2Data.MethodLines[2], - astQueryID, + astQueryID, 0, ).Return(metaResult2Data.SimilarityID, nil) sourceProviderMock := mock_app_source_file.NewMockSourceFileRepo(ctrl) sourceProviderMock.EXPECT(). @@ -117,7 +117,7 @@ func TestMetadataFactory_GetMetadataForQueryAndResult(t *testing.T) { methodLineProvider.EXPECT(). GetMethodLinesByPath(scanID, metaQuery.QueryID). Return(methodLinesResult, nil) - metadata := NewMetadataFactory(astQueryIDProviderMock, similarityIDProviderMock, sourceProviderMock, methodLineProvider, tmpDir) + metadata := NewMetadataFactory(astQueryIDProviderMock, similarityIDProviderMock, sourceProviderMock, methodLineProvider, tmpDir, 0) result, err := metadata.GetMetadataRecord(scanID, []*Query{metaQuery}) assert.NoError(t, err) diff --git a/internal/app/metadata/models.go b/internal/app/metadata/models.go index 477d5ee..956f28e 100644 --- a/internal/app/metadata/models.go +++ b/internal/app/metadata/models.go @@ -50,6 +50,7 @@ type ( Filename1, Name1, Line1, Column1, MethodLine1, Filename2, Name2, Line2, Column2, MethodLine2, QueryID string + SimIDVersion int } SimilarityCalculationResult struct { diff --git a/internal/integration/similarity/similarity_id_calculator.go b/internal/integration/similarity/similarity_id_calculator.go index 5361039..745289e 100644 --- a/internal/integration/similarity/similarity_id_calculator.go +++ b/internal/integration/similarity/similarity_id_calculator.go @@ -1,6 +1,7 @@ package similarity import ( + "fmt" "os" "os/exec" "path/filepath" @@ -18,6 +19,7 @@ type IDProvider interface { filename1, name1, line1, column1, methodLine1, filename2, name2, line2, column2, methodLine2, queryID string, + simIDVersion int, ) (string, error) } @@ -39,19 +41,21 @@ func (e *IDCalculator) Calculate( filename1, name1, line1, column1, methodLine1, filename2, name2, line2, column2, methodLine2, queryID string, + simIDVersion int, ) (string, error) { command := exec.Command( //nolint:gosec e.calculatorCmd, filename1, name1, line1, column1, methodLine1, filename2, name2, line2, column2, methodLine2, queryID, + fmt.Sprint(simIDVersion), ) out, err := command.Output() if err != nil { return "", errors.Wrapf( err, - "failed running command file1=%s name1=%s line1=%s col1=%s method1=%s file2=%s name2=%s line2=%s col2=%s method2=%s query=%s", - filename1, name1, line1, column1, methodLine1, filename2, name2, line2, column2, methodLine2, queryID, + "failed running command file1=%s name1=%s line1=%s col1=%s method1=%s file2=%s name2=%s line2=%s col2=%s method2=%s query=%s simIDVersion=%d", //nolint:lll + filename1, name1, line1, column1, methodLine1, filename2, name2, line2, column2, methodLine2, queryID, simIDVersion, ) } return strings.TrimSpace(string(out)), nil diff --git a/internal/models.go b/internal/models.go index f166cf8..a59c303 100644 --- a/internal/models.go +++ b/internal/models.go @@ -18,6 +18,7 @@ type Args struct { RunTime time.Time QueryMappingFile string NestedTeams bool + SimIDVersion int } type ReportJob struct { diff --git a/internal/process.go b/internal/process.go index 330e5d6..ecec20c 100644 --- a/internal/process.go +++ b/internal/process.go @@ -159,7 +159,14 @@ func RunExport(args *Args) error { } }() - metadataSource := metadata.NewMetadataFactory(astQueryProvider, similarityIDCalculator, sourceRepo, methodLineRepo, metadataTempDir) + metadataSource := metadata.NewMetadataFactory( + astQueryProvider, + similarityIDCalculator, + sourceRepo, + methodLineRepo, + metadataTempDir, + args.SimIDVersion, + ) addErr := addCustomQueryIDs(astQueryProvider, astQueryMappingProvider) if addErr != nil { @@ -495,8 +502,6 @@ func fetchResultsData(client rest.Client, exporter export2.Exporter, resultsProj Str("scans", fmt.Sprintf("%v", triagedScans)). Msg("last scans by project") - log.Info().Msgf("%d results found", len(triagedScans)) - // create and fetch report for each scan go produceReports(triagedScans, reportJobs) @@ -595,6 +600,7 @@ func getTriagedScans(client rest.Client, fromDate, teamName, projectsIds string) if len(*triagedResults) > 0 { output = append(output, TriagedScan{project.ID, project.LastScanID}) } + log.Info().Msgf("fetching %d triaged results found from projectId %d scanId %d", len(*triagedResults), project.ID, project.LastScanID) } // prepare to fetch next page diff --git a/test/mocks/integration/similarity/provider_mock.go b/test/mocks/integration/similarity/provider_mock.go index f26b746..e7a72d0 100644 --- a/test/mocks/integration/similarity/provider_mock.go +++ b/test/mocks/integration/similarity/provider_mock.go @@ -5,6 +5,7 @@ // // mockgen -package mock_integration_similarity -destination test/mocks/integration/similarity/provider_mock.go github.com/checkmarxDev/ast-sast-export/internal/integration/similarity IDProvider // + // Package mock_integration_similarity is a generated GoMock package. package mock_integration_similarity @@ -38,16 +39,16 @@ func (m *MockIDProvider) EXPECT() *MockIDProviderMockRecorder { } // Calculate mocks base method. -func (m *MockIDProvider) Calculate(arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8, arg9, arg10 string) (string, error) { +func (m *MockIDProvider) Calculate(arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8, arg9, arg10 string, arg11 int) (string, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Calculate", arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8, arg9, arg10) + ret := m.ctrl.Call(m, "Calculate", arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8, arg9, arg10, arg11) ret0, _ := ret[0].(string) ret1, _ := ret[1].(error) return ret0, ret1 } // Calculate indicates an expected call of Calculate. -func (mr *MockIDProviderMockRecorder) Calculate(arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8, arg9, arg10 any) *gomock.Call { +func (mr *MockIDProviderMockRecorder) Calculate(arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8, arg9, arg10, arg11 any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Calculate", reflect.TypeOf((*MockIDProvider)(nil).Calculate), arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8, arg9, arg10) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Calculate", reflect.TypeOf((*MockIDProvider)(nil).Calculate), arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8, arg9, arg10, arg11) }