From 64585a6a9685f71f6892269f16ed4afbc479e2a9 Mon Sep 17 00:00:00 2001 From: Assaf Attias <49212512+attiasas@users.noreply.github.com> Date: Wed, 4 Sep 2024 17:57:01 +0300 Subject: [PATCH] Record SARIF results after security commands to upload for GitHub (#138) --- commands/audit/audit.go | 3 +- commands/curation/curationaudit.go | 15 +- commands/curation/curationaudit_test.go | 9 +- commands/enrich/enrich.go | 2 +- commands/scan/buildscan.go | 3 +- commands/scan/dockerscan.go | 11 +- commands/scan/scan.go | 14 +- formats/sarifutils/sarifutils.go | 177 ++++++- formats/sarifutils/sarifutils_test.go | 2 +- formats/sarifutils/test_sarifutils.go | 83 +++- go.mod | 2 +- go.sum | 4 +- .../other/jobSummary/security_section.md | 7 +- .../violations_not_extended_view.md | 2 +- utils/paths.go | 11 +- utils/results.go | 6 +- utils/resultstable.go | 18 +- utils/resultstable_test.go | 2 +- utils/resultwriter.go | 438 ++++++++++++++++-- utils/resultwriter_test.go | 274 ++++++++++- utils/securityJobSummary.go | 158 +++++-- utils/securityJobSummary_test.go | 8 +- utils/utils.go | 36 ++ 23 files changed, 1116 insertions(+), 169 deletions(-) diff --git a/commands/audit/audit.go b/commands/audit/audit.go index 76b56c89..016ff212 100644 --- a/commands/audit/audit.go +++ b/commands/audit/audit.go @@ -140,7 +140,6 @@ func (auditCmd *AuditCommand) Run() (err error) { SetOutputFormat(auditCmd.OutputFormat()). SetPrintExtendedTable(auditCmd.PrintExtendedTable). SetExtraMessages(messages). - SetScanType(services.Dependency). SetSubScansPreformed(auditCmd.ScansToPerform()). PrintScanResults(); err != nil { return @@ -170,7 +169,7 @@ func (auditCmd *AuditCommand) HasViolationContext() bool { // If the current server is entitled for JAS, the advanced security results will be included in the scan results. func RunAudit(auditParams *AuditParams) (results *utils.Results, err error) { // Initialize Results struct - results = utils.NewAuditResults() + results = utils.NewAuditResults(utils.SourceCode) serverDetails, err := auditParams.ServerDetails() if err != nil { return diff --git a/commands/curation/curationaudit.go b/commands/curation/curationaudit.go index 65b988a6..dd63b6c8 100644 --- a/commands/curation/curationaudit.go +++ b/commands/curation/curationaudit.go @@ -4,6 +4,14 @@ import ( "encoding/json" "errors" "fmt" + "net/http" + "os" + "path/filepath" + "regexp" + "sort" + "strings" + "sync" + "golang.org/x/exp/maps" "github.com/jfrog/gofrog/datastructures" @@ -28,13 +36,6 @@ import ( "github.com/jfrog/jfrog-client-go/utils/log" xrayClient "github.com/jfrog/jfrog-client-go/xray" xrayUtils "github.com/jfrog/jfrog-client-go/xray/services/utils" - "net/http" - "os" - "path/filepath" - "regexp" - "sort" - "strings" - "sync" ) const ( diff --git a/commands/curation/curationaudit_test.go b/commands/curation/curationaudit_test.go index a61a5b84..05f29595 100644 --- a/commands/curation/curationaudit_test.go +++ b/commands/curation/curationaudit_test.go @@ -979,7 +979,14 @@ func Test_convertResultsToSummary(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - assert.ElementsMatch(t, tt.expected.Scans, convertResultsToSummary(tt.input).Scans) + summary := convertResultsToSummary(tt.input) + // Sort Blocked base on count (low first) to make the test deterministic + for _, scan := range summary.Scans { + sort.Slice(scan.CuratedPackages.Blocked, func(i, j int) bool { + return len(scan.CuratedPackages.Blocked[i].Packages) < len(scan.CuratedPackages.Blocked[j].Packages) + }) + } + assert.Equal(t, tt.expected, summary) }) } } diff --git a/commands/enrich/enrich.go b/commands/enrich/enrich.go index a2137cc2..badbe2df 100644 --- a/commands/enrich/enrich.go +++ b/commands/enrich/enrich.go @@ -190,7 +190,7 @@ func (enrichCmd *EnrichCommand) Run() (err error) { scanErrors = appendErrorSlice(scanErrors, fileProducerErrors) scanErrors = appendErrorSlice(scanErrors, indexedFileProducerErrors) - scanResults := xrutils.NewAuditResults() + scanResults := xrutils.NewAuditResults(utils.SBOM) scanResults.XrayVersion = xrayVersion scanResults.ScaResults = flatResults diff --git a/commands/scan/buildscan.go b/commands/scan/buildscan.go index f85cf2d0..a1a59148 100644 --- a/commands/scan/buildscan.go +++ b/commands/scan/buildscan.go @@ -149,7 +149,7 @@ func (bsc *BuildScanCommand) runBuildScanAndPrintResults(xrayManager *xray.XrayS XrayDataUrl: buildScanResults.MoreDetailsUrl, }} - scanResults := utils.NewAuditResults() + scanResults := utils.NewAuditResults(utils.Build) scanResults.XrayVersion = xrayVersion scanResults.ScaResults = []*utils.ScaScanResult{{Target: fmt.Sprintf("%s (%s)", params.BuildName, params.BuildNumber), XrayResults: scanResponse}} @@ -160,7 +160,6 @@ func (bsc *BuildScanCommand) runBuildScanAndPrintResults(xrayManager *xray.XrayS SetIncludeLicenses(false). SetIsMultipleRootProject(true). SetPrintExtendedTable(bsc.printExtendedTable). - SetScanType(services.Binary). SetExtraMessages(nil) if bsc.outputFormat != outputFormat.Table { diff --git a/commands/scan/dockerscan.go b/commands/scan/dockerscan.go index 7d260f24..dfb9c59c 100644 --- a/commands/scan/dockerscan.go +++ b/commands/scan/dockerscan.go @@ -96,12 +96,19 @@ func (dsc *DockerScanCommand) Run() (err error) { err = errorutils.CheckError(e) } }() - return dsc.ScanCommand.RunAndRecordResults(func(scanResults *utils.Results) (err error) { + return dsc.ScanCommand.RunAndRecordResults(utils.DockerImage, func(scanResults *utils.Results) (err error) { if scanResults == nil { return } + if scanResults.ScaResults != nil { + for _, result := range scanResults.ScaResults { + result.Name = dsc.imageTag + } + } dsc.analyticsMetricsService.UpdateGeneralEvent(dsc.analyticsMetricsService.CreateXscAnalyticsGeneralEventFinalizeFromAuditResults(scanResults)) - + if err = utils.RecordSarifOutput(scanResults); err != nil { + return + } return utils.RecordSecurityCommandSummary(utils.NewDockerScanSummary( scanResults, dsc.ScanCommand.serverDetails, diff --git a/commands/scan/scan.go b/commands/scan/scan.go index 37bba0c0..a3d913f5 100644 --- a/commands/scan/scan.go +++ b/commands/scan/scan.go @@ -194,7 +194,10 @@ func (scanCmd *ScanCommand) indexFile(filePath string) (*xrayUtils.BinaryGraphNo } func (scanCmd *ScanCommand) Run() (err error) { - return scanCmd.RunAndRecordResults(func(scanResults *utils.Results) error { + return scanCmd.RunAndRecordResults(utils.Binary, func(scanResults *utils.Results) (err error) { + if err = utils.RecordSarifOutput(scanResults); err != nil { + return + } return utils.RecordSecurityCommandSummary(utils.NewBinaryScanSummary( scanResults, scanCmd.serverDetails, @@ -204,7 +207,7 @@ func (scanCmd *ScanCommand) Run() (err error) { }) } -func (scanCmd *ScanCommand) RunAndRecordResults(recordResFunc func(scanResults *utils.Results) error) (err error) { +func (scanCmd *ScanCommand) RunAndRecordResults(cmdType utils.CommandType, recordResFunc func(scanResults *utils.Results) error) (err error) { defer func() { if err != nil { var e *exec.ExitError @@ -220,7 +223,7 @@ func (scanCmd *ScanCommand) RunAndRecordResults(recordResFunc func(scanResults * return err } - scanResults := utils.NewAuditResults() + scanResults := utils.NewAuditResults(cmdType) scanResults.XrayVersion = xrayVersion if scanCmd.analyticsMetricsService != nil { scanResults.MultiScanId = scanCmd.analyticsMetricsService.GetMsi() @@ -323,15 +326,10 @@ func (scanCmd *ScanCommand) RunAndRecordResults(recordResFunc func(scanResults * SetIncludeLicenses(scanCmd.includeLicenses). SetPrintExtendedTable(scanCmd.printExtendedTable). SetIsMultipleRootProject(scanResults.IsMultipleProject()). - SetScanType(services.Binary). PrintScanResults(); err != nil { return } - if err != nil { - return err - } - if err = recordResFunc(scanResults); err != nil { return err } diff --git a/formats/sarifutils/sarifutils.go b/formats/sarifutils/sarifutils.go index e061c4ff..6b183cb5 100644 --- a/formats/sarifutils/sarifutils.go +++ b/formats/sarifutils/sarifutils.go @@ -1,12 +1,10 @@ package sarifutils import ( - "encoding/json" "fmt" "path/filepath" "strings" - "github.com/jfrog/jfrog-client-go/utils" "github.com/jfrog/jfrog-client-go/utils/errorutils" "github.com/owenrumney/go-sarif/v2/sarif" ) @@ -19,12 +17,42 @@ func NewReport() (*sarif.Report, error) { return report, nil } -func ConvertSarifReportToString(report *sarif.Report) (sarifStr string, err error) { - out, err := json.Marshal(report) - if err != nil { - return "", errorutils.CheckError(err) +func CombineReports(reports ...*sarif.Report) (combined *sarif.Report, err error) { + if combined, err = NewReport(); err != nil { + return + } + for _, report := range reports { + for _, run := range report.Runs { + combined.AddRun(run) + } + } + return +} + +func NewPhysicalLocation(physicalPath string) *sarif.PhysicalLocation { + return &sarif.PhysicalLocation{ + ArtifactLocation: &sarif.ArtifactLocation{ + URI: &physicalPath, + }, + } +} + +func NewPhysicalLocationWithRegion(physicalPath string, startRow, endRow, startCol, endCol int) *sarif.PhysicalLocation { + location := NewPhysicalLocation(physicalPath) + location.Region = &sarif.Region{ + StartLine: &startRow, + EndLine: &endRow, + StartColumn: &startCol, + EndColumn: &endCol, + } + return location +} + +func NewLogicalLocation(name, kind string) *sarif.LogicalLocation { + return &sarif.LogicalLocation{ + Name: &name, + Kind: &kind, } - return utils.IndentJson(out), nil } func ReadScanRunsFromFile(fileName string) (sarifRuns []*sarif.Run, err error) { @@ -79,6 +107,19 @@ func isSameLocation(location *sarif.Location, other *sarif.Location) bool { return GetLocationId(location) == GetLocationId(other) } +func GetLogicalLocation(kind string, location *sarif.Location) *sarif.LogicalLocation { + if location == nil { + return nil + } + // Search for a logical location that has the same kind as the location + for _, logicalLocation := range location.LogicalLocations { + if logicalLocation.Kind != nil && *logicalLocation.Kind == kind { + return logicalLocation + } + } + return nil +} + func GetLocationId(location *sarif.Location) string { return fmt.Sprintf("%s:%s:%d:%d:%d:%d", GetLocationFileName(location), @@ -90,6 +131,55 @@ func GetLocationId(location *sarif.Location) string { ) } +func SetRunToolName(toolName string, run *sarif.Run) { + if run.Tool.Driver == nil { + run.Tool.Driver = &sarif.ToolComponent{} + } + run.Tool.Driver.Name = toolName +} + +func SetRunToolFullDescriptionText(txt string, run *sarif.Run) { + if run.Tool.Driver == nil { + run.Tool.Driver = &sarif.ToolComponent{} + } + if run.Tool.Driver.FullDescription == nil { + run.Tool.Driver.FullDescription = sarif.NewMultiformatMessageString(txt) + return + } + run.Tool.Driver.FullDescription.Text = &txt +} + +func SetRunToolFullDescriptionMarkdown(markdown string, run *sarif.Run) { + if run.Tool.Driver == nil { + run.Tool.Driver = &sarif.ToolComponent{} + } + if run.Tool.Driver.FullDescription == nil { + run.Tool.Driver.FullDescription = sarif.NewMarkdownMultiformatMessageString(markdown) + } + run.Tool.Driver.FullDescription.Markdown = &markdown +} + +func GetRunToolFullDescriptionText(run *sarif.Run) string { + if run.Tool.Driver != nil && run.Tool.Driver.FullDescription != nil && run.Tool.Driver.FullDescription.Text != nil { + return *run.Tool.Driver.FullDescription.Text + } + return "" +} + +func GetRunToolFullDescriptionMarkdown(run *sarif.Run) string { + if run.Tool.Driver != nil && run.Tool.Driver.FullDescription != nil && run.Tool.Driver.FullDescription.Markdown != nil { + return *run.Tool.Driver.FullDescription.Markdown + } + return "" +} + +func GetRunToolName(run *sarif.Run) string { + if run.Tool.Driver != nil { + return run.Tool.Driver.Name + } + return "" +} + func GetResultsLocationCount(runs ...*sarif.Run) (count int) { for _, run := range runs { for _, result := range run.Results { @@ -110,7 +200,10 @@ func GetRunsByWorkingDirectory(workingDirectory string, runs ...*sarif.Run) (fil } } return +} +func SetResultMsgMarkdown(markdown string, result *sarif.Result) { + result.Message.Markdown = &markdown } func GetResultMsgText(result *sarif.Result) string { @@ -127,6 +220,34 @@ func GetResultLevel(result *sarif.Result) string { return "" } +func GetResultRuleId(result *sarif.Result) string { + if result.RuleID != nil { + return *result.RuleID + } + return "" +} + +func IsFingerprintsExists(result *sarif.Result) bool { + return len(result.Fingerprints) > 0 +} + +func SetResultFingerprint(algorithm, value string, result *sarif.Result) { + if result.Fingerprints == nil { + result.Fingerprints = make(map[string]interface{}) + } + result.Fingerprints[algorithm] = value +} + +func GetResultLocationSnippets(result *sarif.Result) []string { + var snippets []string + for _, location := range result.Locations { + if snippet := GetLocationSnippet(location); snippet != "" { + snippets = append(snippets, snippet) + } + } + return snippets +} + func GetLocationSnippet(location *sarif.Location) string { region := getLocationRegion(location) if region != nil && region.Snippet != nil { @@ -148,6 +269,31 @@ func GetLocationFileName(location *sarif.Location) string { return "" } +func GetResultFileLocations(result *sarif.Result) []string { + var locations []string + for _, location := range result.Locations { + locations = append(locations, GetLocationFileName(location)) + } + return locations +} + +func ConvertRunsPathsToRelative(runs ...*sarif.Run) { + for _, run := range runs { + for _, result := range run.Results { + for _, location := range result.Locations { + SetLocationFileName(location, GetRelativeLocationFileName(location, run.Invocations)) + } + for _, flows := range result.CodeFlows { + for _, flow := range flows.ThreadFlows { + for _, location := range flow.Locations { + SetLocationFileName(location.Location, GetRelativeLocationFileName(location.Location, run.Invocations)) + } + } + } + } + } +} + func GetRelativeLocationFileName(location *sarif.Location, invocations []*sarif.Invocation) string { wd := "" if len(invocations) > 0 { @@ -227,13 +373,28 @@ func IsResultKindNotPass(result *sarif.Result) bool { return !(result.Kind != nil && *result.Kind == "pass") } -func GetRuleFullDescription(rule *sarif.ReportingDescriptor) string { +func GetRuleFullDescriptionText(rule *sarif.ReportingDescriptor) string { if rule.FullDescription != nil && rule.FullDescription.Text != nil { return *rule.FullDescription.Text } return "" } +func SetRuleShortDescriptionText(value string, rule *sarif.ReportingDescriptor) { + if rule.ShortDescription == nil { + rule.ShortDescription = sarif.NewMultiformatMessageString(value) + return + } + rule.ShortDescription.Text = &value +} + +func GetRuleShortDescriptionText(rule *sarif.ReportingDescriptor) string { + if rule.ShortDescription != nil && rule.ShortDescription.Text != nil { + return *rule.ShortDescription.Text + } + return "" +} + func GetRunRules(run *sarif.Run) []*sarif.ReportingDescriptor { if run != nil && run.Tool.Driver != nil { return run.Tool.Driver.Rules diff --git a/formats/sarifutils/sarifutils_test.go b/formats/sarifutils/sarifutils_test.go index 302cdfd9..6363b515 100644 --- a/formats/sarifutils/sarifutils_test.go +++ b/formats/sarifutils/sarifutils_test.go @@ -544,7 +544,7 @@ func TestGetRuleFullDescription(t *testing.T) { } for _, test := range tests { - assert.Equal(t, test.expectedOutput, GetRuleFullDescription(test.rule)) + assert.Equal(t, test.expectedOutput, GetRuleFullDescriptionText(test.rule)) } } diff --git a/formats/sarifutils/test_sarifutils.go b/formats/sarifutils/test_sarifutils.go index 2de6c19e..6848849a 100644 --- a/formats/sarifutils/test_sarifutils.go +++ b/formats/sarifutils/test_sarifutils.go @@ -2,8 +2,28 @@ package sarifutils import "github.com/owenrumney/go-sarif/v2/sarif" +func CreateRunWithDummyResultsInWd(wd string, results ...*sarif.Result) *sarif.Run { + return createRunWithDummyResults("", results...).WithInvocations([]*sarif.Invocation{sarif.NewInvocation().WithWorkingDirectory(sarif.NewSimpleArtifactLocation(wd))}) +} + func CreateRunWithDummyResults(results ...*sarif.Result) *sarif.Run { - run := sarif.NewRunWithInformationURI("", "") + return createRunWithDummyResults("", results...) +} + +func CreateDummyDriver(toolName, infoURI string, rules ...*sarif.ReportingDescriptor) *sarif.ToolComponent { + return &sarif.ToolComponent{ + Name: toolName, + InformationURI: &infoURI, + Rules: rules, + } +} + +func CreateRunNameWithResults(toolName string, results ...*sarif.Result) *sarif.Run { + return createRunWithDummyResults(toolName, results...) +} + +func createRunWithDummyResults(toolName string, results ...*sarif.Result) *sarif.Run { + run := sarif.NewRunWithInformationURI(toolName, "") for _, result := range results { if result.RuleID != nil { run.AddRule(*result.RuleID) @@ -24,15 +44,60 @@ func CreateRunWithDummyResultAndRuleProperties(property, value string, result *s return run } -func CreateResultWithLocations(msg, ruleId, level string, locations ...*sarif.Location) *sarif.Result { +func CreateDummyResultInPath(fileName string) *sarif.Result { + return CreateResultWithOneLocation(fileName, 0, 0, 0, 0, "snippet", "rule", "level") +} + +func CreateDummyResult(markdown, msg, ruleId, level string) *sarif.Result { return &sarif.Result{ - Message: *sarif.NewTextMessage(msg), - Locations: locations, - Level: &level, - RuleID: &ruleId, + Message: *sarif.NewTextMessage(msg).WithMarkdown(markdown), + Level: &level, + RuleID: &ruleId, } } +func CreateResultWithDummyLocationAmdProperty(fileName, property, value string) *sarif.Result { + resultWithLocation := CreateDummyResultInPath(fileName) + resultWithLocation.Properties = map[string]interface{}{property: value} + return resultWithLocation +} + +func CreateResultWithLocations(msg, ruleId, level string, locations ...*sarif.Location) *sarif.Result { + result := CreateDummyResult("", msg, ruleId, level) + result.Locations = locations + return result +} + +func CreateDummyResultWithFingerprint(markdown, msg, algorithm, value string, locations ...*sarif.Location) *sarif.Result { + result := CreateDummyResult(markdown, msg, "rule", "level") + if result.RuleIndex == nil { + result.RuleIndex = newUintPtr(0) + } + result.Locations = locations + result.Fingerprints = map[string]interface{}{algorithm: value} + return result +} + +func newUintPtr(v uint) *uint { + return &v +} + +func CreateDummyResultWithPathAndLogicalLocation(fileName, logicalName, kind, property, value string) *sarif.Result { + result := CreateDummyResult("", "", "rule", "level") + result.Locations = append(result.Locations, CreateDummyLocationWithPathAndLogicalLocation(fileName, logicalName, kind, property, value)) + return result +} + +func CreateDummyLocationWithPathAndLogicalLocation(fileName, logicalName, kind, property, value string) *sarif.Location { + location := CreateDummyLocationInPath(fileName) + location.LogicalLocations = append(location.LogicalLocations, CreateLogicalLocationWithProperty(logicalName, kind, property, value)) + return location +} + +func CreateDummyLocationInPath(fileName string) *sarif.Location { + return CreateLocation(fileName, 0, 0, 0, 0, "snippet") +} + func CreateLocation(fileName string, startLine, startCol, endLine, endCol int, snippet string) *sarif.Location { return &sarif.Location{ PhysicalLocation: &sarif.PhysicalLocation{ @@ -46,6 +111,12 @@ func CreateLocation(fileName string, startLine, startCol, endLine, endCol int, s } } +func CreateLogicalLocationWithProperty(name, kind, property, value string) *sarif.LogicalLocation { + location := sarif.NewLogicalLocation().WithName(name).WithKind(kind) + location.Properties = map[string]interface{}{property: value} + return location +} + func CreateDummyPassingResult(ruleId string) *sarif.Result { kind := "pass" return &sarif.Result{ diff --git a/go.mod b/go.mod index 42caa94e..ef59500e 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/jfrog/froggit-go v1.16.1 github.com/jfrog/gofrog v1.7.5 github.com/jfrog/jfrog-apps-config v1.0.1 - github.com/jfrog/jfrog-cli-core/v2 v2.55.6 + github.com/jfrog/jfrog-cli-core/v2 v2.55.7 github.com/jfrog/jfrog-client-go v1.46.1 github.com/magiconair/properties v1.8.7 github.com/owenrumney/go-sarif/v2 v2.3.0 diff --git a/go.sum b/go.sum index 854d47fa..8ebca8bf 100644 --- a/go.sum +++ b/go.sum @@ -898,8 +898,8 @@ github.com/jfrog/gofrog v1.7.5 h1:dFgtEDefJdlq9cqTRoe09RLxS5Bxbe1Ev5+E6SmZHcg= github.com/jfrog/gofrog v1.7.5/go.mod h1:jyGiCgiqSSR7k86hcUSu67XVvmvkkgWTmPsH25wI298= github.com/jfrog/jfrog-apps-config v1.0.1 h1:mtv6k7g8A8BVhlHGlSveapqf4mJfonwvXYLipdsOFMY= github.com/jfrog/jfrog-apps-config v1.0.1/go.mod h1:8AIIr1oY9JuH5dylz2S6f8Ym2MaadPLR6noCBO4C22w= -github.com/jfrog/jfrog-cli-core/v2 v2.55.6 h1:3tQuEdYgS2q7fkrrSG66OnO0S998FXGaY9BVsxSLst4= -github.com/jfrog/jfrog-cli-core/v2 v2.55.6/go.mod h1:DPO5BfWAeOByahFMMy+PcjmbPlcyoRy7Bf2C5sGKVi0= +github.com/jfrog/jfrog-cli-core/v2 v2.55.7 h1:V4dO2FMNIH49lov3dMj3jYRg8KBTG7hyhHI8ftYByf8= +github.com/jfrog/jfrog-cli-core/v2 v2.55.7/go.mod h1:DPO5BfWAeOByahFMMy+PcjmbPlcyoRy7Bf2C5sGKVi0= github.com/jfrog/jfrog-client-go v1.46.1 h1:ExqOF8ClOG9LO3vbm6jTIwQHHhprbu8lxB2RrM6mMI0= github.com/jfrog/jfrog-client-go v1.46.1/go.mod h1:UCu2JNBfMp9rypEmCL84DCooG79xWIHVadZQR3Ab+BQ= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= diff --git a/tests/testdata/other/jobSummary/security_section.md b/tests/testdata/other/jobSummary/security_section.md index 517122ca..685a59ce 100644 --- a/tests/testdata/other/jobSummary/security_section.md +++ b/tests/testdata/other/jobSummary/security_section.md @@ -1,8 +1,9 @@ -

🔒 Security Summary

 -#### Curation Audit +

🔒 Curation Audit

 + | Audit Summary | Project name | Audit Details | |--------|--------|---------| | failed.svg | /application1 | 
Total Number of resolved packages: 6
🟢 Approved packages: 3
🔴 Blocked packages: 3
Violated Policy: cvss_score, Condition: cvss score higher than 4.0 (2)📦 npm://test:2.0.0
📦 npm://underscore:1.0.0
Violated Policy: Malicious, Condition: Malicious package (1)📦 npm://lodash:1.0.0
| | passed.svg | /application2 | 
Total Number of resolved packages: 3
| -| failed.svg | /application3 | 
Total Number of resolved packages: 5
🟢 Approved packages: 4
🔴 Blocked packages: 1
Violated Policy: Aged, Condition: Package is aged (1)📦 npm://test:1.0.0
|
\ No newline at end of file +| failed.svg | /application3 | 
Total Number of resolved packages: 5
🟢 Approved packages: 4
🔴 Blocked packages: 1
Violated Policy: Aged, Condition: Package is aged (1)📦 npm://test:1.0.0
| +
\ No newline at end of file diff --git a/tests/testdata/other/jobSummary/violations_not_extended_view.md b/tests/testdata/other/jobSummary/violations_not_extended_view.md index 0702399a..1aead66d 100644 --- a/tests/testdata/other/jobSummary/violations_not_extended_view.md +++ b/tests/testdata/other/jobSummary/violations_not_extended_view.md @@ -1 +1 @@ -
watch: watch1

26 Policy Violations:	20 Security	2 Operational	1 License	3 Secrets

🐸 Unlock detailed findings
\ No newline at end of file +
watch: watch1

26 Policy Violations:	20 Security	2 Operational	1 License	3 Secrets

🐸 Unlock detailed findings
\ No newline at end of file diff --git a/utils/paths.go b/utils/paths.go index 918d1b9b..cafe460c 100644 --- a/utils/paths.go +++ b/utils/paths.go @@ -1,9 +1,6 @@ package utils import ( - // #nosec G505 -- Not in use for secrets. - "crypto/sha1" - "encoding/hex" "github.com/jfrog/jfrog-cli-core/v2/utils/coreutils" "github.com/jfrog/jfrog-cli-security/utils/techutils" "os" @@ -58,13 +55,7 @@ func getProjectPathHash() (string, error) { if err != nil { return "", err } - // #nosec G401 -- Not a secret hash. - hasher := sha1.New() - _, err = hasher.Write([]byte(workingDir)) - if err != nil { - return "", err - } - return hex.EncodeToString(hasher.Sum(nil)), nil + return Sha1Hash(workingDir) } func GetCurationPipCacheFolder() (string, error) { diff --git a/utils/results.go b/utils/results.go index b1404099..bcee459b 100644 --- a/utils/results.go +++ b/utils/results.go @@ -10,6 +10,7 @@ import ( ) type Results struct { + ResultType CommandType ScaResults []*ScaScanResult XrayVersion string ScansErr error @@ -19,8 +20,8 @@ type Results struct { MultiScanId string } -func NewAuditResults() *Results { - return &Results{ExtendedScanResults: &ExtendedScanResults{}} +func NewAuditResults(resultType CommandType) *Results { + return &Results{ResultType: resultType, ExtendedScanResults: &ExtendedScanResults{}} } func (r *Results) GetScaScansXrayResults() (results []services.ScanResponse) { @@ -92,6 +93,7 @@ func (r *Results) CountScanResultsFindings(includeVulnerabilities, includeViolat type ScaScanResult struct { // Could be working directory (audit), file path (binary scan) or build name+number (build scan) Target string `json:"Target"` + Name string `json:"Name,omitempty"` Technology techutils.Technology `json:"Technology,omitempty"` XrayResults []services.ScanResponse `json:"XrayResults,omitempty"` Descriptors []string `json:"Descriptors,omitempty"` diff --git a/utils/resultstable.go b/utils/resultstable.go index bf87b948..06623eac 100644 --- a/utils/resultstable.go +++ b/utils/resultstable.go @@ -38,13 +38,13 @@ const ( // In case one (or more) of the violations contains the field FailBuild set to true, CliError with exit code 3 will be returned. // Set printExtended to true to print fields with 'extended' tag. // If the scan argument is set to true, print the scan tables. -func PrintViolationsTable(violations []services.Violation, results *Results, multipleRoots, printExtended bool, scanType services.ScanType) error { +func PrintViolationsTable(violations []services.Violation, results *Results, multipleRoots, printExtended bool) error { securityViolationsRows, licenseViolationsRows, operationalRiskViolationsRows, err := prepareViolations(violations, results, multipleRoots, true, true) if err != nil { return err } // Print tables, if scan is true; print the scan tables. - if scanType == services.Binary { + if results.ResultType.IsTargetBinary() { err = coreutils.PrintTable(formats.ConvertToVulnerabilityScanTableRow(securityViolationsRows), "Security Violations", "No security violations were found", printExtended) if err != nil { return err @@ -192,13 +192,13 @@ func prepareViolations(violations []services.Violation, results *Results, multip // In case multipleRoots is true, the field Component will show the root of each impact path, otherwise it will show the root's child. // Set printExtended to true to print fields with 'extended' tag. // If the scan argument is set to true, print the scan tables. -func PrintVulnerabilitiesTable(vulnerabilities []services.Vulnerability, results *Results, multipleRoots, printExtended bool, scanType services.ScanType) error { +func PrintVulnerabilitiesTable(vulnerabilities []services.Vulnerability, results *Results, multipleRoots, printExtended bool, scanType CommandType) error { vulnerabilitiesRows, err := prepareVulnerabilities(vulnerabilities, results, multipleRoots, true, true) if err != nil { return err } - if scanType == services.Binary { + if scanType.IsTargetBinary() { return coreutils.PrintTable(formats.ConvertToVulnerabilityScanTableRow(vulnerabilitiesRows), "Vulnerable Components", "✨ No vulnerable components were found ✨", printExtended) } var emptyTableMessage string @@ -300,12 +300,12 @@ func getJfrogResearchPriority(vulnerabilityOrViolation formats.VulnerabilityOrVi // In case multipleRoots is true, the field Component will show the root of each impact path, otherwise it will show the root's child. // Set printExtended to true to print fields with 'extended' tag. // If the scan argument is set to true, print the scan tables. -func PrintLicensesTable(licenses []services.License, printExtended bool, scanType services.ScanType) error { +func PrintLicensesTable(licenses []services.License, printExtended bool, scanType CommandType) error { licensesRows, err := PrepareLicenses(licenses) if err != nil { return err } - if scanType == services.Binary { + if scanType.IsTargetBinary() { return coreutils.PrintTable(formats.ConvertToLicenseScanTableRow(licensesRows), "Licenses", "No licenses were found", printExtended) } return coreutils.PrintTable(formats.ConvertToLicenseTableRow(licensesRows), "Licenses", "No licenses were found", printExtended) @@ -398,7 +398,7 @@ func prepareIacs(iacs []*sarif.Run, isTable bool) []formats.SourceCodeRow { for _, iacResult := range iacRun.Results { scannerDescription := "" if rule, err := iacRun.GetRuleById(*iacResult.RuleID); err == nil { - scannerDescription = sarifutils.GetRuleFullDescription(rule) + scannerDescription = sarifutils.GetRuleFullDescriptionText(rule) } currSeverity, err := severityutils.ParseSeverity(sarifutils.GetResultLevel(iacResult), true) if err != nil { @@ -452,7 +452,7 @@ func prepareSast(sasts []*sarif.Run, isTable bool) []formats.SourceCodeRow { for _, sastResult := range sastRun.Results { scannerDescription := "" if rule, err := sastRun.GetRuleById(*sastResult.RuleID); err == nil { - scannerDescription = sarifutils.GetRuleFullDescription(rule) + scannerDescription = sarifutils.GetRuleFullDescriptionText(rule) } currSeverity, err := severityutils.ParseSeverity(sarifutils.GetResultLevel(sastResult), true) if err != nil { @@ -935,7 +935,7 @@ func getCveApplicabilityField(cveId string, applicabilityScanResults []*sarif.Ru var applicabilityStatuses []jasutils.ApplicabilityStatus for _, applicabilityRun := range applicabilityScanResults { if rule, _ := applicabilityRun.GetRuleById(jasutils.CveToApplicabilityRuleId(cveId)); rule != nil { - applicability.ScannerDescription = sarifutils.GetRuleFullDescription(rule) + applicability.ScannerDescription = sarifutils.GetRuleFullDescriptionText(rule) status := getApplicabilityStatusFromRule(rule) if status != "" { applicabilityStatuses = append(applicabilityStatuses, status) diff --git a/utils/resultstable_test.go b/utils/resultstable_test.go index ea41591c..d0e25ba2 100644 --- a/utils/resultstable_test.go +++ b/utils/resultstable_test.go @@ -27,7 +27,7 @@ func TestPrintViolationsTable(t *testing.T) { } for _, test := range tests { - err := PrintViolationsTable(test.violations, NewAuditResults(), false, true, services.Binary) + err := PrintViolationsTable(test.violations, NewAuditResults(Binary), false, true) assert.NoError(t, err) if CheckIfFailBuild([]services.ScanResponse{{Violations: test.violations}}) { err = NewFailBuildError() diff --git a/utils/resultwriter.go b/utils/resultwriter.go index cad4a751..331d7724 100644 --- a/utils/resultwriter.go +++ b/utils/resultwriter.go @@ -4,6 +4,9 @@ import ( "bytes" "encoding/json" "fmt" + "os" + "path/filepath" + "regexp" "strconv" "strings" @@ -25,11 +28,24 @@ import ( ) const ( - BaseDocumentationURL = "https://docs.jfrog-applications.jfrog.io/jfrog-security-features/" + BaseDocumentationURL = "https://docs.jfrog-applications.jfrog.io/jfrog-security-features/" + CurrentWorkflowNameEnvVar = "GITHUB_WORKFLOW" + CurrentWorkflowRunNumberEnvVar = "GITHUB_RUN_NUMBER" + CurrentWorkflowWorkspaceEnvVar = "GITHUB_WORKSPACE" + + MissingCveScore = "0" + maxPossibleCve = 10.0 + + // #nosec G101 -- Not credentials. + patchedBinarySecretScannerToolName = "JFrog Binary Secrets Scanner" + jfrogFingerprintAlgorithmName = "jfrogFingerprintHash" ) -const MissingCveScore = "0" -const maxPossibleCve = 10.0 +var ( + GithubBaseWorkflowDir = filepath.Join(".github", "workflows") + dockerJasLocationPathPattern = regexp.MustCompile(`.*[\\/](?P[^\\/]+)[\\/](?P[0-9a-fA-F]+)[\\/](?P.*)`) + dockerScaComponentNamePattern = regexp.MustCompile(`(?P[^__]+)__(?P[0-9a-fA-F]+)\.tar`) +) type ResultsWriter struct { // The scan results. @@ -38,9 +54,9 @@ type ResultsWriter struct { simpleJsonError []formats.SimpleJsonError // Format The output format. format format.OutputFormat - // IncludeVulnerabilities If true, include all vulnerabilities as part of the output. Else, include violations only. + // IncludeVulnerabilities If true, include all vulnerabilities as part of the output. includeVulnerabilities bool - // + // If true, include violations as part of the output. hasViolationContext bool // IncludeLicenses If true, also include license violations as part of the output. includeLicenses bool @@ -48,8 +64,6 @@ type ResultsWriter struct { isMultipleRoots bool // PrintExtended, If true, show extended results. printExtended bool - // The scanType (binary,dependency) - scanType services.ScanType // For table format - show table only for the given subScansPreformed subScansPreformed []SubScanType // Messages - Option array of messages, to be displayed if the format is Table @@ -77,11 +91,6 @@ func (rw *ResultsWriter) SetOutputFormat(f format.OutputFormat) *ResultsWriter { return rw } -func (rw *ResultsWriter) SetScanType(scanType services.ScanType) *ResultsWriter { - rw.scanType = scanType - return rw -} - func (rw *ResultsWriter) SetSimpleJsonError(jsonErrors []formats.SimpleJsonError) *ResultsWriter { rw.simpleJsonError = jsonErrors return rw @@ -148,41 +157,41 @@ func (rw *ResultsWriter) printScanResultsTables() (err error) { printMessage(coreutils.PrintTitle("The full scan results are available here: ") + coreutils.PrintLink(resultsPath)) } log.Output() - if shouldPrintTable(rw.subScansPreformed, ScaScan, rw.scanType) { + if shouldPrintTable(rw.subScansPreformed, ScaScan, rw.results.ResultType) { if rw.hasViolationContext { - if err = PrintViolationsTable(violations, rw.results, rw.isMultipleRoots, rw.printExtended, rw.scanType); err != nil { + if err = PrintViolationsTable(violations, rw.results, rw.isMultipleRoots, rw.printExtended); err != nil { return } } if rw.includeVulnerabilities { - if err = PrintVulnerabilitiesTable(vulnerabilities, rw.results, rw.isMultipleRoots, rw.printExtended, rw.scanType); err != nil { + if err = PrintVulnerabilitiesTable(vulnerabilities, rw.results, rw.isMultipleRoots, rw.printExtended, rw.results.ResultType); err != nil { return } } if rw.includeLicenses { - if err = PrintLicensesTable(licenses, rw.printExtended, rw.scanType); err != nil { + if err = PrintLicensesTable(licenses, rw.printExtended, rw.results.ResultType); err != nil { return } } } - if shouldPrintTable(rw.subScansPreformed, SecretsScan, rw.scanType) { + if shouldPrintTable(rw.subScansPreformed, SecretsScan, rw.results.ResultType) { if err = PrintSecretsTable(rw.results.ExtendedScanResults.SecretsScanResults, rw.results.ExtendedScanResults.EntitledForJas); err != nil { return } } - if shouldPrintTable(rw.subScansPreformed, IacScan, rw.scanType) { + if shouldPrintTable(rw.subScansPreformed, IacScan, rw.results.ResultType) { if err = PrintIacTable(rw.results.ExtendedScanResults.IacScanResults, rw.results.ExtendedScanResults.EntitledForJas); err != nil { return } } - if !shouldPrintTable(rw.subScansPreformed, SastScan, rw.scanType) { + if !shouldPrintTable(rw.subScansPreformed, SastScan, rw.results.ResultType) { return nil } return PrintSastTable(rw.results.ExtendedScanResults.SastScanResults, rw.results.ExtendedScanResults.EntitledForJas) } -func shouldPrintTable(requestedScans []SubScanType, subScan SubScanType, scanType services.ScanType) bool { - if scanType == services.Binary && (subScan == IacScan || subScan == SastScan) { +func shouldPrintTable(requestedScans []SubScanType, subScan SubScanType, scanType CommandType) bool { + if scanType.IsTargetBinary() && (subScan == IacScan || subScan == SastScan) { return false } return len(requestedScans) == 0 || slices.Contains(requestedScans, subScan) @@ -201,7 +210,7 @@ func printMessage(message string) { log.Output("💬" + message) } -func GenereateSarifReportFromResults(results *Results, isMultipleRoots, includeLicenses bool, allowedLicenses []string) (report *sarif.Report, err error) { +func GenerateSarifReportFromResults(results *Results, isMultipleRoots, includeLicenses bool, allowedLicenses []string) (report *sarif.Report, err error) { report, err = sarifutils.NewReport() if err != nil { return @@ -211,11 +220,10 @@ func GenereateSarifReportFromResults(results *Results, isMultipleRoots, includeL return } - report.Runs = append(report.Runs, xrayRun) - report.Runs = append(report.Runs, results.ExtendedScanResults.ApplicabilityScanResults...) - report.Runs = append(report.Runs, results.ExtendedScanResults.IacScanResults...) - report.Runs = append(report.Runs, results.ExtendedScanResults.SecretsScanResults...) - report.Runs = append(report.Runs, results.ExtendedScanResults.SastScanResults...) + report.Runs = append(report.Runs, patchRunsToPassIngestionRules(ScaScan, results, xrayRun)...) + report.Runs = append(report.Runs, patchRunsToPassIngestionRules(IacScan, results, results.ExtendedScanResults.IacScanResults...)...) + report.Runs = append(report.Runs, patchRunsToPassIngestionRules(SecretsScan, results, results.ExtendedScanResults.SecretsScanResults...)...) + report.Runs = append(report.Runs, patchRunsToPassIngestionRules(SastScan, results, results.ExtendedScanResults.SastScanResults...)...) return } @@ -225,10 +233,10 @@ func convertXrayResponsesToSarifRun(results *Results, isMultipleRoots, includeLi if err != nil { return } - xrayRun := sarif.NewRunWithInformationURI("JFrog Xray SCA", BaseDocumentationURL+"sca") + xrayRun := sarif.NewRunWithInformationURI("JFrog Xray Scanner", BaseDocumentationURL+"sca") xrayRun.Tool.Driver.Version = &results.XrayVersion if len(xrayJson.Vulnerabilities) > 0 || len(xrayJson.SecurityViolations) > 0 || len(xrayJson.LicensesViolations) > 0 { - if err = extractXrayIssuesToSarifRun(xrayRun, xrayJson); err != nil { + if err = extractXrayIssuesToSarifRun(results, xrayRun, xrayJson); err != nil { return } } @@ -236,26 +244,26 @@ func convertXrayResponsesToSarifRun(results *Results, isMultipleRoots, includeLi return } -func extractXrayIssuesToSarifRun(run *sarif.Run, xrayJson formats.SimpleJsonResults) error { +func extractXrayIssuesToSarifRun(results *Results, run *sarif.Run, xrayJson formats.SimpleJsonResults) error { for _, vulnerability := range xrayJson.Vulnerabilities { - if err := addXrayCveIssueToSarifRun(vulnerability, run); err != nil { + if err := addXrayCveIssueToSarifRun(results, vulnerability, run); err != nil { return err } } for _, violation := range xrayJson.SecurityViolations { - if err := addXrayCveIssueToSarifRun(violation, run); err != nil { + if err := addXrayCveIssueToSarifRun(results, violation, run); err != nil { return err } } for _, license := range xrayJson.LicensesViolations { - if err := addXrayLicenseViolationToSarifRun(license, run); err != nil { + if err := addXrayLicenseViolationToSarifRun(results, license, run); err != nil { return err } } return nil } -func addXrayCveIssueToSarifRun(issue formats.VulnerabilityOrViolationRow, run *sarif.Run) (err error) { +func addXrayCveIssueToSarifRun(results *Results, issue formats.VulnerabilityOrViolationRow, run *sarif.Run) (err error) { maxCveScore, err := findMaxCVEScore(issue.Cves) if err != nil { return @@ -271,6 +279,7 @@ func addXrayCveIssueToSarifRun(issue formats.VulnerabilityOrViolationRow, run *s cveId := GetIssueIdentifier(issue.Cves, issue.IssueId) markdownDescription := getSarifTableDescription(formattedDirectDependencies, maxCveScore, issue.Applicable, issue.FixedVersions) addXrayIssueToSarifRun( + results.ResultType, cveId, issue.ImpactedDependencyName, issue.ImpactedDependencyVersion, @@ -286,12 +295,13 @@ func addXrayCveIssueToSarifRun(issue formats.VulnerabilityOrViolationRow, run *s return } -func addXrayLicenseViolationToSarifRun(license formats.LicenseRow, run *sarif.Run) (err error) { +func addXrayLicenseViolationToSarifRun(results *Results, license formats.LicenseRow, run *sarif.Run) (err error) { formattedDirectDependencies, err := getDirectDependenciesFormatted(license.Components) if err != nil { return } addXrayIssueToSarifRun( + results.ResultType, license.LicenseKey, license.ImpactedDependencyName, license.ImpactedDependencyVersion, @@ -307,21 +317,29 @@ func addXrayLicenseViolationToSarifRun(license formats.LicenseRow, run *sarif.Ru return } -func addXrayIssueToSarifRun(issueId, impactedDependencyName, impactedDependencyVersion string, severity severityutils.Severity, severityScore, summary, title, markdownDescription string, components []formats.ComponentRow, location *sarif.Location, run *sarif.Run) { +func addXrayIssueToSarifRun(resultType CommandType, issueId, impactedDependencyName, impactedDependencyVersion string, severity severityutils.Severity, severityScore, summary, title, markdownDescription string, components []formats.ComponentRow, location *sarif.Location, run *sarif.Run) { // Add rule if not exists ruleId := getXrayIssueSarifRuleId(impactedDependencyName, impactedDependencyVersion, issueId) if rule, _ := run.GetRuleById(ruleId); rule == nil { addXrayRule(ruleId, title, severityScore, summary, markdownDescription, run) } // Add result for each component - for _, directDependency := range components { msg := getXrayIssueSarifHeadline(directDependency.Name, directDependency.Version, issueId) if result := run.CreateResultForRule(ruleId).WithMessage(sarif.NewTextMessage(msg)).WithLevel(severityutils.SeverityToSarifSeverityLevel(severity).String()); location != nil { + if resultType == DockerImage { + algorithm, layer := getLayerContentFromComponentId(directDependency.Name) + if layer != "" { + logicalLocation := sarifutils.NewLogicalLocation(layer, "layer") + if algorithm != "" { + logicalLocation.Properties = map[string]interface{}{"algorithm": algorithm} + } + location.LogicalLocations = append(location.LogicalLocations, logicalLocation) + } + } result.AddLocation(location) } } - } func getDescriptorFullPath(tech techutils.Technology, run *sarif.Run) (string, error) { @@ -459,7 +477,7 @@ func getXrayIssueSarifRuleId(depName, version, key string) string { } func getXrayIssueSarifHeadline(depName, version, key string) string { - return fmt.Sprintf("[%s] %s %s", key, depName, version) + return strings.TrimSpace(fmt.Sprintf("[%s] %s %s", key, depName, version)) } func getXrayLicenseSarifHeadline(depName, version, key string) string { @@ -520,6 +538,319 @@ func findMaxCVEScore(cves []formats.CveRow) (string, error) { return strCve, nil } +func patchRules(subScanType SubScanType, cmdResults *Results, rules ...*sarif.ReportingDescriptor) (patched []*sarif.ReportingDescriptor) { + patched = []*sarif.ReportingDescriptor{} + for _, rule := range rules { + // Github code scanning ingestion rules rejects rules without help content. + // Patch by transferring the full description to the help field. + if rule.Help == nil && rule.FullDescription != nil { + rule.Help = rule.FullDescription + } + // SARIF1001 - if both 'id' and 'name' are present, they must be different. If they are identical, the tool must omit the 'name' property. + if rule.Name != nil && rule.ID == *rule.Name { + rule.Name = nil + } + if cmdResults.ResultType.IsTargetBinary() && subScanType == SecretsScan { + // Patch the rule name in case of binary scan + sarifutils.SetRuleShortDescriptionText(fmt.Sprintf("[Secret in Binary found] %s", sarifutils.GetRuleShortDescriptionText(rule)), rule) + } + patched = append(patched, rule) + } + return +} + +func patchResults(subScanType SubScanType, cmdResults *Results, run *sarif.Run, results ...*sarif.Result) (patched []*sarif.Result) { + patched = []*sarif.Result{} + for _, result := range results { + if len(result.Locations) == 0 { + // Github code scanning ingestion rules rejects results without locations. + // Patch by removing results without locations. + log.Debug(fmt.Sprintf("[%s] Removing result [ruleId=%s] without locations: %s", subScanType.String(), sarifutils.GetResultRuleId(result), sarifutils.GetResultMsgText(result))) + continue + } + if cmdResults.ResultType.IsTargetBinary() { + var markdown string + if subScanType == SecretsScan { + markdown = getSecretInBinaryMarkdownMsg(cmdResults, result) + } else { + markdown = getScaInBinaryMarkdownMsg(cmdResults, result) + } + sarifutils.SetResultMsgMarkdown(markdown, result) + // For Binary scans, override the physical location if applicable (after data already used for markdown) + convertBinaryPhysicalLocations(cmdResults, run, result) + // Calculate the fingerprints if not exists + if !sarifutils.IsFingerprintsExists(result) { + if err := calculateResultFingerprints(cmdResults.ResultType, run, result); err != nil { + log.Warn(fmt.Sprintf("Failed to calculate the fingerprint for result [ruleId=%s]: %s", sarifutils.GetResultRuleId(result), err.Error())) + } + } + } + patched = append(patched, result) + } + return patched +} + +func patchRunsToPassIngestionRules(subScanType SubScanType, cmdResults *Results, runs ...*sarif.Run) []*sarif.Run { + // Since we run in temp directories files should be relative + // Patch by converting the file paths to relative paths according to the invocations + convertPaths(cmdResults.ResultType, subScanType, runs...) + for _, run := range runs { + if cmdResults.ResultType.IsTargetBinary() && subScanType == SecretsScan { + // Patch the tool name in case of binary scan + sarifutils.SetRunToolName(patchedBinarySecretScannerToolName, run) + } + run.Tool.Driver.Rules = patchRules(subScanType, cmdResults, run.Tool.Driver.Rules...) + run.Results = patchResults(subScanType, cmdResults, run, run.Results...) + } + return runs +} + +func convertPaths(commandType CommandType, subScanType SubScanType, runs ...*sarif.Run) { + // Convert base on invocation for source code + sarifutils.ConvertRunsPathsToRelative(runs...) + if !(commandType == DockerImage && subScanType == SecretsScan) { + return + } + for _, run := range runs { + for _, result := range run.Results { + // For Docker secret scan, patch the logical location if not exists + patchDockerSecretLocations(result) + } + } +} + +// Patch the URI to be the file path from sha// +// Extract the layer from the location URI, adds it as a logical location kind "layer" +func patchDockerSecretLocations(result *sarif.Result) { + for _, location := range result.Locations { + algorithm, layerHash, relativePath := getLayerContentFromPath(sarifutils.GetLocationFileName(location)) + if layerHash != "" { + // Set Logical location kind "layer" with the layer hash + logicalLocation := sarifutils.NewLogicalLocation(layerHash, "layer") + if algorithm != "" { + logicalLocation.Properties = sarif.Properties(map[string]interface{}{"algorithm": algorithm}) + } + location.LogicalLocations = append(location.LogicalLocations, logicalLocation) + } + if relativePath != "" { + sarifutils.SetLocationFileName(location, relativePath) + } + } +} + +func convertBinaryPhysicalLocations(cmdResults *Results, run *sarif.Run, result *sarif.Result) { + if patchedLocation := getPatchedBinaryLocation(cmdResults, run); patchedLocation != "" { + for _, location := range result.Locations { + // Patch the location - Reset the uri and region + location.PhysicalLocation = sarifutils.NewPhysicalLocation(patchedLocation) + } + } +} + +func getPatchedBinaryLocation(cmdResults *Results, run *sarif.Run) (patchedLocation string) { + if cmdResults.ResultType == DockerImage { + if patchedLocation = getDockerfileLocationIfExists(run); patchedLocation != "" { + return + } + } + return getWorkflowFileLocationIfExists() +} + +func getDockerfileLocationIfExists(run *sarif.Run) string { + potentialLocations := []string{filepath.Clean("Dockerfile"), sarifutils.GetFullLocationFileName("Dockerfile", run.Invocations)} + for _, location := range potentialLocations { + if exists, err := fileutils.IsFileExists(location, false); err == nil && exists { + return location + } + } + if workspace := os.Getenv(CurrentWorkflowWorkspaceEnvVar); workspace != "" { + if exists, err := fileutils.IsFileExists(filepath.Join(workspace, "Dockerfile"), false); err == nil && exists { + return filepath.Join(workspace, "Dockerfile") + } + } + return "" +} + +func getGithubWorkflowsDirIfExists() string { + if exists, err := fileutils.IsDirExists(GithubBaseWorkflowDir, false); err == nil && exists { + return GithubBaseWorkflowDir + } + if workspace := os.Getenv(CurrentWorkflowWorkspaceEnvVar); workspace != "" { + if exists, err := fileutils.IsDirExists(filepath.Join(workspace, GithubBaseWorkflowDir), false); err == nil && exists { + return filepath.Join(workspace, GithubBaseWorkflowDir) + } + } + return "" +} + +func getWorkflowFileLocationIfExists() (location string) { + workflowName := os.Getenv(CurrentWorkflowNameEnvVar) + if workflowName == "" { + return + } + workflowsDir := getGithubWorkflowsDirIfExists() + if workflowsDir == "" { + return + } + currentWd, err := os.Getwd() + if err != nil { + log.Warn(fmt.Sprintf("Failed to get the current working directory to get workflow file location: %s", err.Error())) + return + } + // Check if exists in the .github/workflows directory as file name or in the content, return the file path or empty string + if files, err := fileutils.ListFiles(workflowsDir, false); err == nil && len(files) > 0 { + for _, file := range files { + if strings.Contains(file, workflowName) { + return strings.TrimPrefix(file, currentWd) + } + } + for _, file := range files { + if content, err := fileutils.ReadFile(file); err == nil && strings.Contains(string(content), workflowName) { + return strings.TrimPrefix(file, currentWd) + } + } + } + return +} + +func getSecretInBinaryMarkdownMsg(cmdResults *Results, result *sarif.Result) string { + if cmdResults.ResultType != Binary && cmdResults.ResultType != DockerImage { + return "" + } + content := "🔒 Found Secrets in Binary" + if cmdResults.ResultType == DockerImage { + content += " docker" + } + content += " scanning:" + return content + getBaseBinaryDescriptionMarkdown(SecretsScan, cmdResults, result) +} + +func getScaInBinaryMarkdownMsg(cmdResults *Results, result *sarif.Result) string { + return sarifutils.GetResultMsgText(result) + getBaseBinaryDescriptionMarkdown(ScaScan, cmdResults, result) +} + +func getBaseBinaryDescriptionMarkdown(subScanType SubScanType, cmdResults *Results, result *sarif.Result) (content string) { + // If in github action, add the workflow name and run number + if workflowLocation := getWorkflowFileLocationIfExists(); workflowLocation != "" { + content += fmt.Sprintf("\nGithub Actions Workflow: %s", workflowLocation) + } + if os.Getenv(CurrentWorkflowRunNumberEnvVar) != "" { + content += fmt.Sprintf("\nRun: %s", os.Getenv(CurrentWorkflowRunNumberEnvVar)) + } + // If is docker image, add the image tag + if cmdResults.ResultType == DockerImage { + if imageTag := getDockerImageTag(cmdResults); imageTag != "" { + content += fmt.Sprintf("\nImage: %s", imageTag) + } + } + var location *sarif.Location + if len(result.Locations) > 0 { + location = result.Locations[0] + } + return content + getBinaryLocationMarkdownString(cmdResults.ResultType, subScanType, location) +} + +func getDockerImageTag(cmdResults *Results) string { + if cmdResults.ResultType != DockerImage || len(cmdResults.ScaResults) == 0 { + return "" + } + for _, scaResults := range cmdResults.ScaResults { + if scaResults.Name != "" { + return scaResults.Name + } + } + return filepath.Base(cmdResults.ScaResults[0].Target) +} + +// If command is docker prepare the markdown string for the location: +// * Layer: +// * Filepath: +// * Evidence: +func getBinaryLocationMarkdownString(commandType CommandType, subScanType SubScanType, location *sarif.Location) (content string) { + if location == nil { + return "" + } + if commandType == DockerImage { + if layer, algorithm := getDockerLayer(location); layer != "" { + if algorithm != "" { + content += fmt.Sprintf("\nLayer (%s): %s", algorithm, layer) + } else { + content += fmt.Sprintf("\nLayer: %s", layer) + } + } + } + if subScanType != SecretsScan { + return + } + if locationFilePath := sarifutils.GetLocationFileName(location); locationFilePath != "" { + content += fmt.Sprintf("\nFilepath: %s", locationFilePath) + } + if snippet := sarifutils.GetLocationSnippet(location); snippet != "" { + content += fmt.Sprintf("\nEvidence: %s", snippet) + } + return +} + +func getDockerLayer(location *sarif.Location) (layer, algorithm string) { + // If location has logical location with kind "layer" return it + if logicalLocation := sarifutils.GetLogicalLocation("layer", location); logicalLocation != nil && logicalLocation.Name != nil { + layer = *logicalLocation.Name + if algorithmValue, ok := logicalLocation.Properties["algorithm"].(string); ok { + algorithm = algorithmValue + } + return + } + return +} + +// Match: +// Extract algorithm, hash and relative path +func getLayerContentFromPath(content string) (algorithm string, layerHash string, relativePath string) { + matches := dockerJasLocationPathPattern.FindStringSubmatch(content) + if len(matches) == 0 { + return + } + algorithm = matches[dockerJasLocationPathPattern.SubexpIndex("algorithm")] + layerHash = matches[dockerJasLocationPathPattern.SubexpIndex("hash")] + relativePath = matches[dockerJasLocationPathPattern.SubexpIndex("relativePath")] + return +} + +// Match: ://:/ +// Extract algorithm and hash +func getLayerContentFromComponentId(componentId string) (algorithm string, layerHash string) { + matches := dockerScaComponentNamePattern.FindStringSubmatch(componentId) + if len(matches) == 0 { + return + } + algorithm = matches[dockerScaComponentNamePattern.SubexpIndex("algorithm")] + layerHash = matches[dockerScaComponentNamePattern.SubexpIndex("hash")] + return +} + +// According to the SARIF specification: +// To determine whether a result from a subsequent run is logically the same as a result from the baseline, +// there must be a way to use information contained in the result to construct a stable identifier for the result. We refer to this identifier as a fingerprint. +// A result management system SHOULD construct a fingerprint by using information contained in the SARIF file such as: +// The name of the tool that produced the result, the rule id, the file system path to the analysis target... +func calculateResultFingerprints(resultType CommandType, run *sarif.Run, result *sarif.Result) error { + if !resultType.IsTargetBinary() { + return nil + } + ids := []string{sarifutils.GetRunToolName(run), sarifutils.GetResultRuleId(result)} + for _, location := range sarifutils.GetResultFileLocations(result) { + ids = append(ids, strings.ReplaceAll(location, string(filepath.Separator), "/")) + } + ids = append(ids, sarifutils.GetResultLocationSnippets(result)...) + // Calculate the hash value and set the fingerprint to the result + hashValue, err := Md5Hash(ids...) + if err != nil { + return err + } + sarifutils.SetResultFingerprint(jfrogFingerprintAlgorithmName, hashValue, result) + return nil +} + // Splits scan responses into aggregated lists of violations, vulnerabilities and licenses. func SplitScanResults(results []*ScaScanResult) ([]services.Violation, []services.Vulnerability, []services.License) { var violations []services.Violation @@ -546,7 +877,7 @@ func writeJsonResults(results *Results) (resultsPath string, err error) { err = e } }() - bytesRes, err := JSONMarshal(&results) + bytesRes, err := JSONMarshalNotEscaped(&results) if errorutils.CheckError(err) != nil { return } @@ -563,7 +894,20 @@ func writeJsonResults(results *Results) (resultsPath string, err error) { return } -func JSONMarshal(t interface{}) ([]byte, error) { +func WriteSarifResultsAsString(report *sarif.Report, escape bool) (sarifStr string, err error) { + var out []byte + if escape { + out, err = json.Marshal(report) + } else { + out, err = JSONMarshalNotEscaped(report) + } + if err != nil { + return "", errorutils.CheckError(err) + } + return clientUtils.IndentJson(out), nil +} + +func JSONMarshalNotEscaped(t interface{}) ([]byte, error) { buffer := &bytes.Buffer{} encoder := json.NewEncoder(buffer) encoder.SetEscapeHTML(false) @@ -572,7 +916,7 @@ func JSONMarshal(t interface{}) ([]byte, error) { } func PrintJson(output interface{}) error { - results, err := JSONMarshal(output) + results, err := JSONMarshalNotEscaped(output) if err != nil { return errorutils.CheckError(err) } @@ -581,11 +925,11 @@ func PrintJson(output interface{}) error { } func PrintSarif(results *Results, isMultipleRoots, includeLicenses bool) error { - sarifReport, err := GenereateSarifReportFromResults(results, isMultipleRoots, includeLicenses, nil) + sarifReport, err := GenerateSarifReportFromResults(results, isMultipleRoots, includeLicenses, nil) if err != nil { return err } - sarifFile, err := sarifutils.ConvertSarifReportToString(sarifReport) + sarifFile, err := WriteSarifResultsAsString(sarifReport, false) if err != nil { return err } @@ -770,7 +1114,11 @@ func getSecuritySummaryFindings(cves []services.Cve, issueId string, components } if len(cves) == 0 { // XRAY-ID, no scanners for them - uniqueFindings[jasutils.NotCovered.String()] += 1 + status := jasutils.NotScanned + if len(applicableRuns) > 0 { + status = jasutils.NotCovered + } + uniqueFindings[status.String()] += 1 } return uniqueFindings } diff --git a/utils/resultwriter_test.go b/utils/resultwriter_test.go index 5f8652b0..d009c777 100644 --- a/utils/resultwriter_test.go +++ b/utils/resultwriter_test.go @@ -1,6 +1,7 @@ package utils import ( + "fmt" "os" "path/filepath" "sort" @@ -11,6 +12,8 @@ import ( "github.com/jfrog/jfrog-cli-security/formats/sarifutils" "github.com/jfrog/jfrog-cli-security/utils/jasutils" "github.com/jfrog/jfrog-cli-security/utils/techutils" + "github.com/jfrog/jfrog-client-go/utils/io/fileutils" + clientTests "github.com/jfrog/jfrog-client-go/utils/tests" "github.com/jfrog/jfrog-client-go/xray/services" "github.com/owenrumney/go-sarif/v2/sarif" "github.com/stretchr/testify/assert" @@ -391,7 +394,7 @@ func TestConvertXrayScanToSimpleJson(t *testing.T) { } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - results := NewAuditResults() + results := NewAuditResults(SourceCode) scaScanResult := ScaScanResult{XrayResults: []services.ScanResponse{tc.result}} results.ScaResults = append(results.ScaResults, &scaScanResult) output, err := ConvertXrayScanToSimpleJson(results, false, tc.includeLicenses, true, tc.allowedLicenses) @@ -436,7 +439,7 @@ func TestJSONMarshall(t *testing.T) { for _, tc := range testCases { t.Run(tc.testName, func(t *testing.T) { - printedString, err := JSONMarshal(tc.resultString) + printedString, err := JSONMarshalNotEscaped(tc.resultString) assert.NoError(t, err) assert.Equal(t, tc.expectedResult, string(printedString)) }) @@ -573,6 +576,273 @@ func TestGetSummary(t *testing.T) { } } +func TestGetLayerContentFromComponentId(t *testing.T) { + testCases := []struct { + name string + path string + expectedAlgorithm string + expectedLayerHash string + }{ + { + name: "Valid path", + path: "sha256__cedb364ef937c7e51179d8e514bdd98644bac5fdc82a45d784ef91afe4bc647e.tar", + expectedAlgorithm: "sha256", + expectedLayerHash: "cedb364ef937c7e51179d8e514bdd98644bac5fdc82a45d784ef91afe4bc647e", + }, + { + name: "Invalid path - not hex", + path: "sha256__NOT_HEX.tar", + }, + { + name: "Invalid path - no algorithm", + path: "_cedb364ef937c7e51179d8e514bdd98644bac5fdc82a45d784ef91afe4bc647e.tar", + }, + { + name: "Invalid path - no suffix", + path: "sha256__cedb364ef937c7e51179d8e514bdd98644bac5fdc82a45d784ef91afe4bc647e", + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + algorithm, layerHash := getLayerContentFromComponentId(tc.path) + assert.Equal(t, tc.expectedAlgorithm, algorithm) + assert.Equal(t, tc.expectedLayerHash, layerHash) + }) + } +} + +func preparePatchTestEnv(t *testing.T) (string, string, func()) { + currentWd, err := os.Getwd() + assert.NoError(t, err) + wd, cleanUpTempDir := tests.CreateTempDirWithCallbackAndAssert(t) + cleanUpWd := clientTests.ChangeDirWithCallback(t, currentWd, wd) + dockerfileDir := filepath.Join(wd, "DockerfileDir") + err = fileutils.CreateDirIfNotExist(dockerfileDir) + // Prepare env content + assert.NoError(t, err) + createDummyDockerfile(t, dockerfileDir) + createDummyGithubWorkflow(t, dockerfileDir) + createDummyGithubWorkflow(t, wd) + return wd, dockerfileDir, func() { + cleanUpWd() + cleanUpTempDir() + } +} + +func createDummyGithubWorkflow(t *testing.T, baseDir string) { + assert.NoError(t, fileutils.CreateDirIfNotExist(filepath.Join(baseDir, GithubBaseWorkflowDir))) + assert.NoError(t, os.WriteFile(filepath.Join(baseDir, GithubBaseWorkflowDir, "workflowFile.yml"), []byte("workflow name"), 0644)) +} + +func createDummyDockerfile(t *testing.T, baseDir string) { + assert.NoError(t, os.WriteFile(filepath.Join(baseDir, "Dockerfile"), []byte("Dockerfile data"), 0644)) +} + +func TestPatchRunsToPassIngestionRules(t *testing.T) { + wd, dockerfileDir, cleanUp := preparePatchTestEnv(t) + defer cleanUp() + + testCases := []struct { + name string + cmdResult *Results + subScan SubScanType + withEnvVars bool + withDockerfile bool + input []*sarif.Run + expectedResults []*sarif.Run + }{ + { + name: "No runs", + cmdResult: &Results{ResultType: DockerImage, ScaResults: []*ScaScanResult{{Name: "dockerImage:imageVersion"}}}, + subScan: SecretsScan, + input: []*sarif.Run{}, + expectedResults: []*sarif.Run{}, + }, + { + name: "Build scan - SCA", + cmdResult: &Results{ResultType: Build, ScaResults: []*ScaScanResult{{Name: "buildName (buildNumber)"}}}, + subScan: ScaScan, + input: []*sarif.Run{ + sarifutils.CreateRunWithDummyResultsInWd(wd, sarifutils.CreateDummyResultInPath(fmt.Sprintf("file://%s", filepath.Join(wd, "dir", "file")))), + }, + expectedResults: []*sarif.Run{ + sarifutils.CreateRunWithDummyResultsInWd(wd, sarifutils.CreateDummyResultInPath(filepath.Join("dir", "file"))), + }, + }, + { + name: "Docker image scan - SCA", + cmdResult: &Results{ResultType: DockerImage, ScaResults: []*ScaScanResult{{Name: "dockerImage:imageVersion"}}}, + subScan: ScaScan, + input: []*sarif.Run{ + sarifutils.CreateRunWithDummyResultAndRuleProperties("applicability", "applicable", sarifutils.CreateDummyResultWithPathAndLogicalLocation("sha256__f752cb05a39e65f231a3c47c2e08cbeac1c15e4daff0188cb129c12a3ea3049d", "f752cb05a39e65f231a3c47c2e08cbeac1c15e4daff0188cb129c12a3ea3049d", "layer", "algorithm", "sha256").WithMessage(sarif.NewTextMessage("some-msg"))). + WithInvocations([]*sarif.Invocation{ + sarif.NewInvocation().WithWorkingDirectory(sarif.NewSimpleArtifactLocation(wd)), + }, + ), + sarifutils.CreateRunWithDummyResultsInWd(wd, + sarifutils.CreateDummyResultWithPathAndLogicalLocation("sha256__f752cb05a39e65f231a3c47c2e08cbeac1c15e4daff0188cb129c12a3ea3049d", "f752cb05a39e65f231a3c47c2e08cbeac1c15e4daff0188cb129c12a3ea3049d", "layer", "algorithm", "sha256").WithMessage(sarif.NewTextMessage("some-msg")), + ), + }, + expectedResults: []*sarif.Run{ + sarifutils.CreateRunWithDummyResultAndRuleProperties("applicability", "applicable", + sarifutils.CreateDummyResultWithFingerprint("some-msg\nImage: dockerImage:imageVersion\nLayer (sha256): f752cb05a39e65f231a3c47c2e08cbeac1c15e4daff0188cb129c12a3ea3049d", "some-msg", jfrogFingerprintAlgorithmName, "9522c1d915eef55b4a0dc9e160bf5dc7", + sarifutils.CreateDummyLocationWithPathAndLogicalLocation("sha256__f752cb05a39e65f231a3c47c2e08cbeac1c15e4daff0188cb129c12a3ea3049d", "f752cb05a39e65f231a3c47c2e08cbeac1c15e4daff0188cb129c12a3ea3049d", "layer", "algorithm", "sha256"), + ), + ).WithInvocations([]*sarif.Invocation{ + sarif.NewInvocation().WithWorkingDirectory(sarif.NewSimpleArtifactLocation(wd)), + }), + sarifutils.CreateRunWithDummyResultsInWd(wd, + sarifutils.CreateDummyResultWithFingerprint("some-msg\nImage: dockerImage:imageVersion\nLayer (sha256): f752cb05a39e65f231a3c47c2e08cbeac1c15e4daff0188cb129c12a3ea3049d", "some-msg", jfrogFingerprintAlgorithmName, "9522c1d915eef55b4a0dc9e160bf5dc7", + sarifutils.CreateDummyLocationWithPathAndLogicalLocation("sha256__f752cb05a39e65f231a3c47c2e08cbeac1c15e4daff0188cb129c12a3ea3049d", "f752cb05a39e65f231a3c47c2e08cbeac1c15e4daff0188cb129c12a3ea3049d", "layer", "algorithm", "sha256"), + ), + ), + }, + }, + { + name: "Docker image scan - with env vars", + cmdResult: &Results{ResultType: DockerImage, ScaResults: []*ScaScanResult{{Name: "dockerImage:imageVersion"}}}, + subScan: ScaScan, + withEnvVars: true, + input: []*sarif.Run{ + sarifutils.CreateRunWithDummyResultsInWd(wd, + sarifutils.CreateDummyResultWithPathAndLogicalLocation("sha256__f752cb05a39e65f231a3c47c2e08cbeac1c15e4daff0188cb129c12a3ea3049d", "f752cb05a39e65f231a3c47c2e08cbeac1c15e4daff0188cb129c12a3ea3049d", "layer", "algorithm", "sha256").WithMessage(sarif.NewTextMessage("some-msg")), + // No location, should be removed in the output + sarifutils.CreateDummyResult("some-markdown", "some-other-msg", "rule", "level"), + ), + }, + expectedResults: []*sarif.Run{ + sarifutils.CreateRunWithDummyResultsInWd(wd, + sarifutils.CreateDummyResultWithFingerprint(fmt.Sprintf("some-msg\nGithub Actions Workflow: %s\nRun: 123\nImage: dockerImage:imageVersion\nLayer (sha256): f752cb05a39e65f231a3c47c2e08cbeac1c15e4daff0188cb129c12a3ea3049d", filepath.Join(GithubBaseWorkflowDir, "workflowFile.yml")), "some-msg", jfrogFingerprintAlgorithmName, "eda26ae830c578197aeda65a82d7f093", + sarifutils.CreateDummyLocationWithPathAndLogicalLocation("", "f752cb05a39e65f231a3c47c2e08cbeac1c15e4daff0188cb129c12a3ea3049d", "layer", "algorithm", "sha256").WithPhysicalLocation( + sarif.NewPhysicalLocation().WithArtifactLocation(sarif.NewSimpleArtifactLocation(filepath.Join(GithubBaseWorkflowDir, "workflowFile.yml"))), + ), + ), + ), + }, + }, + { + name: "Docker image scan - with Dockerfile in wd", + cmdResult: &Results{ResultType: DockerImage, ScaResults: []*ScaScanResult{{Name: "dockerImage:imageVersion"}}}, + subScan: ScaScan, + withEnvVars: true, + withDockerfile: true, + input: []*sarif.Run{ + sarifutils.CreateRunWithDummyResultsInWd(dockerfileDir, + sarifutils.CreateDummyResultWithPathAndLogicalLocation("sha256__f752cb05a39e65f231a3c47c2e08cbeac1c15e4daff0188cb129c12a3ea3049d", "f752cb05a39e65f231a3c47c2e08cbeac1c15e4daff0188cb129c12a3ea3049d", "layer", "algorithm", "sha256").WithMessage(sarif.NewTextMessage("some-msg")), + ), + }, + expectedResults: []*sarif.Run{ + sarifutils.CreateRunWithDummyResultsInWd(dockerfileDir, + sarifutils.CreateDummyResultWithFingerprint(fmt.Sprintf("some-msg\nGithub Actions Workflow: %s\nRun: 123\nImage: dockerImage:imageVersion\nLayer (sha256): f752cb05a39e65f231a3c47c2e08cbeac1c15e4daff0188cb129c12a3ea3049d", filepath.Join(GithubBaseWorkflowDir, "workflowFile.yml")), "some-msg", jfrogFingerprintAlgorithmName, "8cbd7268a4d20f2358ba2667ebd18956", + sarifutils.CreateDummyLocationWithPathAndLogicalLocation("", "f752cb05a39e65f231a3c47c2e08cbeac1c15e4daff0188cb129c12a3ea3049d", "layer", "algorithm", "sha256").WithPhysicalLocation( + sarif.NewPhysicalLocation().WithArtifactLocation(sarif.NewSimpleArtifactLocation("Dockerfile")), + ), + ), + ), + }, + }, + { + name: "Docker image scan - Secrets", + cmdResult: &Results{ResultType: DockerImage, ScaResults: []*ScaScanResult{{Name: "dockerImage:imageVersion"}}}, + subScan: SecretsScan, + input: []*sarif.Run{ + sarifutils.CreateRunNameWithResults("some tool name", + sarifutils.CreateDummyResultInPath(fmt.Sprintf("file://%s", filepath.Join(wd, "unpacked", "filesystem", "blobs", "sha1", "9e88ea9de1b44baba5e96a79e33e4af64334b2bf129e838e12f6dae71b5c86f0", "usr", "src", "app", "server", "index.js"))), + ).WithInvocations([]*sarif.Invocation{ + sarif.NewInvocation().WithWorkingDirectory(sarif.NewSimpleArtifactLocation(wd)), + }), + }, + expectedResults: []*sarif.Run{ + { + Tool: sarif.Tool{ + Driver: sarifutils.CreateDummyDriver(patchedBinarySecretScannerToolName, "", &sarif.ReportingDescriptor{ + ID: "rule", + ShortDescription: sarif.NewMultiformatMessageString("[Secret in Binary found] "), + }), + }, + Invocations: []*sarif.Invocation{sarif.NewInvocation().WithWorkingDirectory(sarif.NewSimpleArtifactLocation(wd))}, + Results: []*sarif.Result{ + sarifutils.CreateDummyResultWithFingerprint(fmt.Sprintf("🔒 Found Secrets in Binary docker scanning:\nImage: dockerImage:imageVersion\nLayer (sha1): 9e88ea9de1b44baba5e96a79e33e4af64334b2bf129e838e12f6dae71b5c86f0\nFilepath: %s\nEvidence: snippet", filepath.Join("usr", "src", "app", "server", "index.js")), "", jfrogFingerprintAlgorithmName, "dee156c9fd75a4237102dc8fb29277a2", + sarifutils.CreateDummyLocationWithPathAndLogicalLocation(filepath.Join("usr", "src", "app", "server", "index.js"), "9e88ea9de1b44baba5e96a79e33e4af64334b2bf129e838e12f6dae71b5c86f0", "layer", "algorithm", "sha1"), + ), + }, + }, + }, + }, + { + name: "Binary scan - SCA", + cmdResult: &Results{ResultType: Binary, ScaResults: []*ScaScanResult{{Target: filepath.Join(wd, "dir", "binary")}}}, + subScan: ScaScan, + input: []*sarif.Run{ + sarifutils.CreateRunWithDummyResultsInWd(wd, + sarifutils.CreateDummyResultInPath(fmt.Sprintf("file://%s", filepath.Join(wd, "dir", "binary"))), + ), + }, + expectedResults: []*sarif.Run{ + sarifutils.CreateRunWithDummyResultsInWd(wd, + sarifutils.CreateDummyResultWithFingerprint("", "", jfrogFingerprintAlgorithmName, "e72a936dc73acbc4283a93230ff9b6e8", sarifutils.CreateDummyLocationInPath(filepath.Join("dir", "binary"))), + ), + }, + }, + { + name: "Audit scan - SCA", + cmdResult: &Results{ResultType: SourceCode, ScaResults: []*ScaScanResult{{Target: wd}}}, + subScan: ScaScan, + input: []*sarif.Run{ + sarifutils.CreateRunWithDummyResultsInWd(wd, + sarifutils.CreateDummyResultInPath(filepath.Join(wd, "Package-Descriptor")), + // No location, should be removed in the output + sarifutils.CreateDummyResult("some-markdown", "some-other-msg", "rule", "level"), + ), + }, + expectedResults: []*sarif.Run{ + sarifutils.CreateRunWithDummyResultsInWd(wd, + sarifutils.CreateDummyResultInPath("Package-Descriptor"), + ), + }, + }, + { + name: "Audit scan - Secrets", + cmdResult: &Results{ResultType: SourceCode, ScaResults: []*ScaScanResult{{Target: wd}}}, + subScan: SecretsScan, + input: []*sarif.Run{ + sarifutils.CreateRunWithDummyResultsInWd(wd, + sarifutils.CreateDummyResultInPath(fmt.Sprintf("file://%s", filepath.Join(wd, "dir", "file"))), + // No location, should be removed in the output + sarifutils.CreateDummyResult("some-markdown", "some-other-msg", "rule", "level"), + ), + }, + expectedResults: []*sarif.Run{ + sarifutils.CreateRunWithDummyResultsInWd(wd, + sarifutils.CreateDummyResultInPath(filepath.Join("dir", "file")), + ), + }, + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + if tc.withEnvVars { + cleanFileEnv := clientTests.SetEnvWithCallbackAndAssert(t, CurrentWorkflowNameEnvVar, "workflow name") + defer cleanFileEnv() + cleanRunNumEnv := clientTests.SetEnvWithCallbackAndAssert(t, CurrentWorkflowRunNumberEnvVar, "123") + defer cleanRunNumEnv() + } else { + // Since the the env are provided by the + cleanFileEnv := clientTests.SetEnvWithCallbackAndAssert(t, CurrentWorkflowNameEnvVar, "") + defer cleanFileEnv() + cleanRunNumEnv := clientTests.SetEnvWithCallbackAndAssert(t, CurrentWorkflowRunNumberEnvVar, "") + defer cleanRunNumEnv() + } + if tc.withDockerfile { + revertWd := clientTests.ChangeDirWithCallback(t, wd, dockerfileDir) + defer revertWd() + } + patchRunsToPassIngestionRules(tc.subScan, tc.cmdResult, tc.input...) + assert.ElementsMatch(t, tc.expectedResults, tc.input) + }) + } +} + func getDummyScaTestResults(vulnerability, violation bool) (responses []services.ScanResponse) { response := services.ScanResponse{} if vulnerability { diff --git a/utils/securityJobSummary.go b/utils/securityJobSummary.go index a1a4be41..210173af 100644 --- a/utils/securityJobSummary.go +++ b/utils/securityJobSummary.go @@ -3,6 +3,7 @@ package utils import ( "errors" "fmt" + "os" "path/filepath" "sort" "strings" @@ -14,18 +15,16 @@ import ( "github.com/jfrog/jfrog-cli-core/v2/utils/config" "github.com/jfrog/jfrog-cli-core/v2/utils/coreutils" "github.com/jfrog/jfrog-cli-security/formats" + "github.com/jfrog/jfrog-cli-security/formats/sarifutils" "github.com/jfrog/jfrog-cli-security/resources" "github.com/jfrog/jfrog-cli-security/utils/jasutils" "github.com/jfrog/jfrog-cli-security/utils/severityutils" + "github.com/jfrog/jfrog-client-go/utils/errorutils" + "github.com/jfrog/jfrog-client-go/utils/log" + "github.com/owenrumney/go-sarif/v2/sarif" ) const ( - Build SecuritySummarySection = "Build-info Scans" - Binary SecuritySummarySection = "Artifact Scans" - Modules SecuritySummarySection = "Source Code Scans" - Docker SecuritySummarySection = "Docker Image Scans" - Curation SecuritySummarySection = "Curation Audit" - PreFormat HtmlTag = "
%s
" ImgTag HtmlTag = "\"%s\"" CenterContent HtmlTag = "
%s
" @@ -33,21 +32,17 @@ const ( Link HtmlTag = "%s" NewLine HtmlTag = "
%s" DetailsWithSummary HtmlTag = "
%s%s
" - DetailsOpenWithSummary HtmlTag = "

%s

%s
" - RedColor HtmlTag = "%s" - OrangeColor HtmlTag = "%s" - GreenColor HtmlTag = "%s" + DetailsOpenWithSummary HtmlTag = "

%s

%s\n
" TabTag HtmlTag = " %s" - ApplicableStatusCount SeverityStatus = "%d Applicable" - NotApplicableStatusCount SeverityStatus = "%d Not Applicable" + ApplicableStatusCount SeverityDisplayStatus = "%d Applicable" + NotApplicableStatusCount SeverityDisplayStatus = "%d Not Applicable" maxWatchesInLine = 4 ) -type SecuritySummarySection string type HtmlTag string -type SeverityStatus string +type SeverityDisplayStatus string func (c HtmlTag) Format(args ...any) string { return fmt.Sprintf(string(c), args...) @@ -57,7 +52,7 @@ func (c HtmlTag) FormatInt(value int) string { return fmt.Sprintf(string(c), fmt.Sprintf("%d", value)) } -func (s SeverityStatus) Format(count int) string { +func (s SeverityDisplayStatus) Format(count int) string { return fmt.Sprintf(string(s), count) } @@ -71,38 +66,38 @@ func getStatusIcon(failed bool) string { type SecurityJobSummary struct{} -func NewCurationSummary(cmdResult formats.ResultsSummary) (summary ScanCommandResultSummary) { - summary.ResultType = Curation - summary.Summary = cmdResult - return -} - -func newResultSummary(cmdResults *Results, section SecuritySummarySection, serverDetails *config.ServerDetails, vulnerabilitiesReqested, violationsReqested bool) (summary ScanCommandResultSummary) { - summary.ResultType = section +func newResultSummary(cmdResults *Results, cmdType CommandType, serverDetails *config.ServerDetails, vulnerabilitiesRequested, violationsRequested bool) (summary ScanCommandResultSummary) { + summary.ResultType = cmdType summary.Args = &ResultSummaryArgs{BaseJfrogUrl: serverDetails.Url} - summary.Summary = ToSummary(cmdResults, vulnerabilitiesReqested, violationsReqested) + summary.Summary = ToSummary(cmdResults, vulnerabilitiesRequested, violationsRequested) return } -func NewBuildScanSummary(cmdResults *Results, serverDetails *config.ServerDetails, vulnerabilitiesReqested, violationsReqested bool, buildName, buildNumber string) (summary ScanCommandResultSummary) { - summary = newResultSummary(cmdResults, Build, serverDetails, vulnerabilitiesReqested, violationsReqested) +func NewBuildScanSummary(cmdResults *Results, serverDetails *config.ServerDetails, vulnerabilitiesRequested, violationsRequested bool, buildName, buildNumber string) (summary ScanCommandResultSummary) { + summary = newResultSummary(cmdResults, Build, serverDetails, vulnerabilitiesRequested, violationsRequested) summary.Args.BuildName = buildName summary.Args.BuildNumbers = []string{buildNumber} return } -func NewDockerScanSummary(cmdResults *Results, serverDetails *config.ServerDetails, vulnerabilitiesReqested, violationsReqested bool, dockerImage string) (summary ScanCommandResultSummary) { - summary = newResultSummary(cmdResults, Docker, serverDetails, vulnerabilitiesReqested, violationsReqested) +func NewDockerScanSummary(cmdResults *Results, serverDetails *config.ServerDetails, vulnerabilitiesRequested, violationsRequested bool, dockerImage string) (summary ScanCommandResultSummary) { + summary = newResultSummary(cmdResults, DockerImage, serverDetails, vulnerabilitiesRequested, violationsRequested) summary.Args.DockerImage = dockerImage return } -func NewBinaryScanSummary(cmdResults *Results, serverDetails *config.ServerDetails, vulnerabilitiesReqested, violationsReqested bool) (summary ScanCommandResultSummary) { - return newResultSummary(cmdResults, Binary, serverDetails, vulnerabilitiesReqested, violationsReqested) +func NewBinaryScanSummary(cmdResults *Results, serverDetails *config.ServerDetails, vulnerabilitiesRequested, violationsRequested bool) (summary ScanCommandResultSummary) { + return newResultSummary(cmdResults, Binary, serverDetails, vulnerabilitiesRequested, violationsRequested) } -func NewAuditScanSummary(cmdResults *Results, serverDetails *config.ServerDetails, vulnerabilitiesReqested, violationsReqested bool) (summary ScanCommandResultSummary) { - return newResultSummary(cmdResults, Modules, serverDetails, vulnerabilitiesReqested, violationsReqested) +func NewAuditScanSummary(cmdResults *Results, serverDetails *config.ServerDetails, vulnerabilitiesRequested, violationsRequested bool) (summary ScanCommandResultSummary) { + return newResultSummary(cmdResults, SourceCode, serverDetails, vulnerabilitiesRequested, violationsRequested) +} + +func NewCurationSummary(cmdResult formats.ResultsSummary) (summary ScanCommandResultSummary) { + summary.ResultType = Curation + summary.Summary = cmdResult + return } type ResultSummaryArgs struct { @@ -133,13 +128,18 @@ func (rsa ResultSummaryArgs) ToArgs(index commandsummary.Index) (args []string) args = append(args, rsa.BuildName) args = append(args, rsa.BuildNumbers...) } else if index == commandsummary.DockerScan { - args = append(args, rsa.DockerImage) + image := rsa.DockerImage + // if user did not provide image tag, add latest + if !strings.Contains(image, ":") { + image += ":latest" + } + args = append(args, image) } return } type ScanCommandResultSummary struct { - ResultType SecuritySummarySection `json:"resultType"` + ResultType CommandType `json:"resultType"` Args *ResultSummaryArgs `json:"args,omitempty"` Summary formats.ResultsSummary `json:"summary"` } @@ -149,12 +149,16 @@ func NewSecurityJobSummary() (js *commandsummary.CommandSummary, err error) { return commandsummary.New(&SecurityJobSummary{}, "security") } -// Record the security command outputs -func RecordSecurityCommandSummary(content ScanCommandResultSummary) (err error) { +func getRecordManager() (manager *commandsummary.CommandSummary, err error) { if !commandsummary.ShouldRecordSummary() { return } - manager, err := NewSecurityJobSummary() + return NewSecurityJobSummary() +} + +// Record the security command outputs +func RecordSecurityCommandSummary(content ScanCommandResultSummary) (err error) { + manager, err := getRecordManager() if err != nil || manager == nil { return } @@ -163,12 +167,64 @@ func RecordSecurityCommandSummary(content ScanCommandResultSummary) (err error) return } updateSummaryNamesToRelativePath(&content.Summary, wd) - if index := getDataIndexFromSection(content.ResultType); index != "" { + if index := getDataIndexFromCommandType(content.ResultType); index != "" { return recordIndexData(manager, content, index) } return manager.Record(content) } +func RecordSarifOutput(cmdResults *Results) (err error) { + manager, err := getRecordManager() + if err != nil || manager == nil { + return + } + extended := true + if !extended && !commandsummary.StaticMarkdownConfig.IsExtendedSummary() { + log.Info("Results can be uploaded to Github security tab automatically by upgrading your JFrog subscription.") + return + } + sarifReport, err := GenerateSarifReportFromResults(cmdResults, true, false, nil) + if err != nil { + return err + } + out, err := JSONMarshalNotEscaped(sarifReport) + if err != nil { + return errorutils.CheckError(err) + } + return manager.RecordWithIndex(out, commandsummary.SarifReport) +} + +func CombineSarifOutputFiles(dataFilePaths []string) (data []byte, err error) { + if len(dataFilePaths) == 0 { + return + } + // Load the content of the files + reports := []*sarif.Report{} + for _, dataFilePath := range dataFilePaths { + if report, e := loadSarifReport(dataFilePath); e != nil { + err = errors.Join(err, e) + } else { + reports = append(reports, report) + } + } + if err != nil { + return + } + combined, err := sarifutils.CombineReports(reports...) + if err != nil { + return + } + return JSONMarshalNotEscaped(combined) +} + +func loadSarifReport(dataFilePath string) (report *sarif.Report, err error) { + fileData, err := os.ReadFile(dataFilePath) + if err != nil { + return + } + return sarif.FromBytes(fileData) +} + func updateSummaryNamesToRelativePath(summary *formats.ResultsSummary, wd string) { for i, scan := range summary.Scans { if scan.Target == "" { @@ -184,15 +240,15 @@ func updateSummaryNamesToRelativePath(summary *formats.ResultsSummary, wd string } } -func getDataIndexFromSection(section SecuritySummarySection) commandsummary.Index { - switch section { +func getDataIndexFromCommandType(cmdType CommandType) commandsummary.Index { + switch cmdType { case Build: return commandsummary.BuildScan case Binary: return commandsummary.BinariesScan - case Modules: + case SourceCode: return commandsummary.BinariesScan - case Docker: + case DockerImage: return commandsummary.DockerScan } // No index for the section @@ -213,11 +269,11 @@ func recordIndexData(manager *commandsummary.CommandSummary, content ScanCommand return } -func newScanCommandResultSummary(resultType SecuritySummarySection, args *ResultSummaryArgs, scans ...formats.ScanSummary) ScanCommandResultSummary { +func newScanCommandResultSummary(resultType CommandType, args *ResultSummaryArgs, scans ...formats.ScanSummary) ScanCommandResultSummary { return ScanCommandResultSummary{ResultType: resultType, Args: args, Summary: formats.ResultsSummary{Scans: scans}} } -func loadContent(dataFiles []string, filterSections ...SecuritySummarySection) ([]formats.ResultsSummary, ResultSummaryArgs, error) { +func loadContent(dataFiles []string, filterSections ...CommandType) ([]formats.ResultsSummary, ResultSummaryArgs, error) { data := []formats.ResultsSummary{} args := ResultSummaryArgs{} for _, dataFilePath := range dataFiles { @@ -282,7 +338,7 @@ func GenerateSecuritySectionMarkdown(curationData []formats.ResultsSummary) (mar return } // Create the markdown content - markdown += fmt.Sprintf("\n\n#### %s\n| Audit Summary | Project name | Audit Details |\n|--------|--------|---------|", Curation) + markdown += "\n\n| Audit Summary | Project name | Audit Details |\n|--------|--------|---------|" for i := range curationData { for _, summary := range curationData[i].Scans { status := getStatusIcon(false) @@ -292,7 +348,7 @@ func GenerateSecuritySectionMarkdown(curationData []formats.ResultsSummary) (mar markdown += fmt.Sprintf("\n| %s | %s | %s |", status, summary.Target, PreFormat.Format(getCurationDetailsString(summary))) } } - markdown = DetailsOpenWithSummary.Format("🔒 Security Summary", markdown) + markdown = "\n" + DetailsOpenWithSummary.Format("🔒 Curation Audit", markdown) return } @@ -327,7 +383,7 @@ func getCurationDetailsString(summary formats.ScanSummary) (content string) { var blocked []blockedPackageByType // Sort the blocked packages by name for _, blockTypeValue := range summary.CuratedPackages.Blocked { - blocked = append(blocked, toBlockedPackgeByType(blockTypeValue)) + blocked = append(blocked, toBlockedPackageByType(blockTypeValue)) } sort.Slice(blocked, func(i, j int) bool { return blocked[i].BlockedType > blocked[j].BlockedType @@ -342,7 +398,7 @@ func getCurationDetailsString(summary formats.ScanSummary) (content string) { return } -func toBlockedPackgeByType(blockTypeValue formats.BlockedPackages) blockedPackageByType { +func toBlockedPackageByType(blockTypeValue formats.BlockedPackages) blockedPackageByType { return blockedPackageByType{BlockedType: formatPolicyAndCond(blockTypeValue.Policy, blockTypeValue.Condition), BlockedSummary: blockTypeValue.Packages} } @@ -563,8 +619,8 @@ func getSeverityStatusesCountString(statusCounts map[string]int) string { return generateSeverityStatusesCountString(getSeverityDisplayStatuses(statusCounts)) } -func getSeverityDisplayStatuses(statusCounts map[string]int) (displayData map[SeverityStatus]int) { - displayData = map[SeverityStatus]int{} +func getSeverityDisplayStatuses(statusCounts map[string]int) (displayData map[SeverityDisplayStatus]int) { + displayData = map[SeverityDisplayStatus]int{} for status, count := range statusCounts { switch status { case jasutils.Applicability.String(): @@ -576,7 +632,7 @@ func getSeverityDisplayStatuses(statusCounts map[string]int) (displayData map[Se return displayData } -func generateSeverityStatusesCountString(displayData map[SeverityStatus]int) string { +func generateSeverityStatusesCountString(displayData map[SeverityDisplayStatus]int) string { if len(displayData) == 0 { return "" } diff --git a/utils/securityJobSummary_test.go b/utils/securityJobSummary_test.go index e447a7fb..d5890907 100644 --- a/utils/securityJobSummary_test.go +++ b/utils/securityJobSummary_test.go @@ -39,7 +39,7 @@ var ( func TestSaveLoadData(t *testing.T) { testDockerScanSummary := ScanCommandResultSummary{ - ResultType: Docker, + ResultType: DockerImage, Args: &ResultSummaryArgs{ BaseJfrogUrl: testPlatformUrl, DockerImage: "dockerImage:version", @@ -133,7 +133,7 @@ func TestSaveLoadData(t *testing.T) { testCases := []struct { name string content []ScanCommandResultSummary - filterSections []SecuritySummarySection + filterSections []CommandType expectedArgs ResultSummaryArgs expectedContent []formats.ResultsSummary }{ @@ -156,7 +156,7 @@ func TestSaveLoadData(t *testing.T) { }, { name: "Multiple scans with filter", - filterSections: []SecuritySummarySection{Curation}, + filterSections: []CommandType{Curation}, content: []ScanCommandResultSummary{testDockerScanSummary, testBinaryScanSummary, testBuildScanSummary, testCurationSummary}, expectedContent: []formats.ResultsSummary{testCurationSummary.Summary}, }, @@ -169,7 +169,7 @@ func TestSaveLoadData(t *testing.T) { // Save the data for i := range testCase.content { updateSummaryNamesToRelativePath(&testCase.content[i].Summary, tempDir) - data, err := JSONMarshal(&testCase.content[i]) + data, err := JSONMarshalNotEscaped(&testCase.content[i]) assert.NoError(t, err) dataFilePath := filepath.Join(tempDir, fmt.Sprintf("data_%s_%d.json", testCase.name, i)) assert.NoError(t, os.WriteFile(dataFilePath, data, 0644)) diff --git a/utils/utils.go b/utils/utils.go index b4aa871e..1b4d2da7 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -1,6 +1,8 @@ package utils import ( + "crypto" + "encoding/hex" "fmt" "strings" ) @@ -38,14 +40,48 @@ func (v ViolationIssueType) String() string { type SubScanType string +const ( + SourceCode CommandType = "source_code" + Binary CommandType = "binary" + DockerImage CommandType = "docker_image" + Build CommandType = "build" + Curation CommandType = "curation" + SBOM CommandType = "SBOM" +) + +type CommandType string + func (s SubScanType) String() string { return string(s) } +func (s CommandType) IsTargetBinary() bool { + return s == Binary || s == DockerImage +} + func GetAllSupportedScans() []SubScanType { return []SubScanType{ScaScan, ContextualAnalysisScan, IacScan, SastScan, SecretsScan} } +func Md5Hash(values ...string) (string, error) { + return toHash(crypto.MD5, values...) +} + +func Sha1Hash(values ...string) (string, error) { + return toHash(crypto.SHA1, values...) +} + +func toHash(hash crypto.Hash, values ...string) (string, error) { + h := hash.New() + for _, ob := range values { + _, err := fmt.Fprint(h, ob) + if err != nil { + return "", err + } + } + return hex.EncodeToString(h.Sum(nil)), nil +} + // map[string]string to []string (key=value format) func ToCommandEnvVars(envVarsMap map[string]string) (converted []string) { converted = make([]string, 0, len(envVarsMap))