Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Run contextual analysis and secret detection in Docker scans #10

Merged
merged 1 commit into from
May 30, 2024
Merged

Run contextual analysis and secret detection in Docker scans #10

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 11 additions & 14 deletions commands/audit/audit.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,16 +3,22 @@ package audit
import (
"errors"
"fmt"
"os"

"github.com/jfrog/jfrog-cli-core/v2/utils/coreutils"
"github.com/jfrog/jfrog-cli-security/scangraph"
"github.com/jfrog/jfrog-cli-security/utils"

"github.com/jfrog/jfrog-cli-security/jas"

"github.com/jfrog/jfrog-cli-security/jas/applicability"
"github.com/jfrog/jfrog-cli-security/jas/runner"
"github.com/jfrog/jfrog-cli-security/jas/secrets"
clientutils "github.com/jfrog/jfrog-client-go/utils"
"github.com/jfrog/jfrog-client-go/utils/log"
"github.com/jfrog/jfrog-client-go/xray"
"github.com/jfrog/jfrog-client-go/xray/services"
xscservices "github.com/jfrog/jfrog-client-go/xsc/services"
"golang.org/x/sync/errgroup"
"os"

xrayutils "github.com/jfrog/jfrog-cli-security/utils"
)
Expand Down Expand Up @@ -182,15 +188,15 @@ func RunAudit(auditParams *AuditParams) (results *xrayutils.Results, err error)
return
}
results.XrayVersion = auditParams.xrayVersion
results.ExtendedScanResults.EntitledForJas, err = isEntitledForJas(xrayManager, auditParams.xrayVersion)
results.ExtendedScanResults.EntitledForJas, err = jas.IsEntitledForJas(xrayManager, auditParams.xrayVersion)
if err != nil {
return
}

errGroup := new(errgroup.Group)
if results.ExtendedScanResults.EntitledForJas {
// Download (if needed) the analyzer manager in a background routine.
errGroup.Go(utils.DownloadAnalyzerManagerIfNeeded)
errGroup.Go(xrayutils.DownloadAnalyzerManagerIfNeeded)
}

results.MultiScanId = auditParams.XrayGraphScanParams().MultiScanId
Expand All @@ -205,16 +211,7 @@ func RunAudit(auditParams *AuditParams) (results *xrayutils.Results, err error)

// Run scanners only if the user is entitled for Advanced Security
if results.ExtendedScanResults.EntitledForJas {
results.JasError = runJasScannersAndSetResults(results, auditParams.DirectDependencies(), serverDetails, auditParams.workingDirs, auditParams.Progress(), auditParams.thirdPartyApplicabilityScan, auditParams.XrayGraphScanParams().MultiScanId)
}
return
}

func isEntitledForJas(xrayManager *xray.XrayServicesManager, xrayVersion string) (entitled bool, err error) {
if e := clientutils.ValidateMinimumVersion(clientutils.Xray, xrayVersion, xrayutils.EntitlementsMinVersion); e != nil {
log.Debug(e)
return
results.JasError = runner.RunJasScannersAndSetResults(results.ExtendedScanResults, results.GetScaScannedTechnologies(), results.GetScaScansXrayResults(), auditParams.DirectDependencies(), serverDetails, auditParams.workingDirs, auditParams.Progress(), auditParams.thirdPartyApplicabilityScan, auditParams.XrayGraphScanParams().MultiScanId, applicability.ApplicabilityScannerType, secrets.SecretsScannerType)
}
entitled, err = xrayManager.IsEntitled(xrayutils.ApplicabilityFeatureId)
return
}
63 changes: 0 additions & 63 deletions commands/audit/jasrunner.go
View file
Open in desktop

This file was deleted.

10 changes: 6 additions & 4 deletions commands/scan/dockerscan.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,16 +3,17 @@ package scan
import (
"bytes"
"fmt"
"os"
"os/exec"
"path/filepath"
"strings"

"github.com/jfrog/jfrog-cli-core/v2/common/spec"
xrayutils "github.com/jfrog/jfrog-cli-security/utils"
clientutils "github.com/jfrog/jfrog-client-go/utils"
"github.com/jfrog/jfrog-client-go/utils/errorutils"
"github.com/jfrog/jfrog-client-go/utils/io/fileutils"
"github.com/jfrog/jfrog-client-go/utils/log"
"os"
"os/exec"
"path/filepath"
"strings"
)

const (
Expand Down Expand Up @@ -81,6 +82,7 @@ func (dsc *DockerScanCommand) Run() (err error) {
Pattern(imageTarPath).
Target(dsc.targetRepoPath).
BuildSpec()).SetThreads(1)
dsc.ScanCommand.SetRunJasScans(true)
err = dsc.setCredentialEnvsForIndexerApp()
if err != nil {
return errorutils.CheckError(err)
Expand Down
83 changes: 71 additions & 12 deletions commands/scan/scan.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,12 +10,19 @@ import (
"regexp"
"strings"

"golang.org/x/exp/maps"
"golang.org/x/exp/slices"

"github.com/jfrog/jfrog-cli-security/jas"
"github.com/jfrog/jfrog-cli-security/jas/applicability"
"github.com/jfrog/jfrog-cli-security/jas/runner"
"github.com/jfrog/jfrog-cli-security/jas/secrets"
"github.com/jfrog/jfrog-cli-security/scangraph"
xrayUtils "github.com/jfrog/jfrog-client-go/xray/services/utils"
"golang.org/x/sync/errgroup"

"github.com/jfrog/gofrog/parallel"
"github.com/jfrog/jfrog-cli-core/v2/common/format"
outputFormat "github.com/jfrog/jfrog-cli-core/v2/common/format"
"github.com/jfrog/jfrog-cli-core/v2/common/spec"
"github.com/jfrog/jfrog-cli-core/v2/utils/config"
"github.com/jfrog/jfrog-cli-core/v2/utils/coreutils"
Expand All @@ -35,8 +42,9 @@ type FileContext func(string) parallel.TaskFunc
type indexFileHandlerFunc func(file string)

type ScanInfo struct {
Target string
Result *services.ScanResponse
Target string
Result *services.ScanResponse
ExtendedScanResults *utils.ExtendedScanResults
}

const (
Expand All @@ -52,7 +60,7 @@ type ScanCommand struct {
// The location of the downloaded Xray indexer binary on the local file system.
indexerPath string
indexerTempDir string
outputFormat outputFormat.OutputFormat
outputFormat format.OutputFormat
projectKey string
minSeverityFilter string
watches []string
Expand All @@ -63,6 +71,7 @@ type ScanCommand struct {
bypassArchiveLimits bool
fixableOnly bool
progress ioUtils.ProgressMgr
commandSupportsJAS bool
}

func (scanCmd *ScanCommand) SetMinSeverityFilter(minSeverityFilter string) *ScanCommand {
Expand All @@ -75,6 +84,11 @@ func (scanCmd *ScanCommand) SetFixableOnly(fixable bool) *ScanCommand {
return scanCmd
}

func (scanCmd *ScanCommand) SetRunJasScans(run bool) *ScanCommand {
scanCmd.commandSupportsJAS = run
return scanCmd
}

func (scanCmd *ScanCommand) SetProgress(progress ioUtils.ProgressMgr) {
scanCmd.progress = progress
}
Expand All @@ -84,7 +98,7 @@ func (scanCmd *ScanCommand) SetThreads(threads int) *ScanCommand {
return scanCmd
}

func (scanCmd *ScanCommand) SetOutputFormat(format outputFormat.OutputFormat) *ScanCommand {
func (scanCmd *ScanCommand) SetOutputFormat(format format.OutputFormat) *ScanCommand {
scanCmd.outputFormat = format
return scanCmd
}
Expand Down Expand Up @@ -182,6 +196,16 @@ func (scanCmd *ScanCommand) Run() (err error) {
return err
}

scanResults := xrutils.NewAuditResults()
scanResults.XrayVersion = xrayVersion

scanResults.ExtendedScanResults.EntitledForJas, err = jas.IsEntitledForJas(xrayManager, xrayVersion)
errGroup := new(errgroup.Group)
if scanResults.ExtendedScanResults.EntitledForJas {
// Download (if needed) the analyzer manager in a background routine.
errGroup.Go(xrutils.DownloadAnalyzerManagerIfNeeded)
}

// Validate Xray minimum version for graph scan command
err = clientutils.ValidateMinimumVersion(clientutils.Xray, xrayVersion, scangraph.GraphScanMinXrayVersion)
if err != nil {
Expand Down Expand Up @@ -226,16 +250,20 @@ func (scanCmd *ScanCommand) Run() (err error) {
fileCollectingErrorsQueue := clientutils.NewErrorsQueue(1)
// Start walking on the filesystem to "produce" files that match the given pattern
// while the consumer uses the indexer to index those files.
scanCmd.prepareScanTasks(fileProducerConsumer, indexedFileProducerConsumer, resultsArr, fileProducerErrors, indexedFileProducerErrors, fileCollectingErrorsQueue, xrayVersion)
scanCmd.prepareScanTasks(fileProducerConsumer, indexedFileProducerConsumer, scanResults.ExtendedScanResults.EntitledForJas, resultsArr, fileProducerErrors, indexedFileProducerErrors, fileCollectingErrorsQueue, xrayVersion)
scanCmd.performScanTasks(fileProducerConsumer, indexedFileProducerConsumer)

// Handle results
flatResults := []xrutils.ScaScanResult{}

for _, arr := range resultsArr {
for _, res := range arr {
flatResults = append(flatResults, xrutils.ScaScanResult{Target: res.Target, XrayResults: []services.ScanResponse{*res.Result}})
scanResults.ExtendedScanResults.ApplicabilityScanResults = append(scanResults.ExtendedScanResults.ApplicabilityScanResults, res.ExtendedScanResults.ApplicabilityScanResults...)
scanResults.ExtendedScanResults.SecretsScanResults = append(scanResults.ExtendedScanResults.SecretsScanResults, res.ExtendedScanResults.SecretsScanResults...)
}
}

if scanCmd.progress != nil {
if err = scanCmd.progress.Quit(); err != nil {
return err
Expand All @@ -251,10 +279,13 @@ func (scanCmd *ScanCommand) Run() (err error) {
scanErrors = appendErrorSlice(scanErrors, fileProducerErrors)
scanErrors = appendErrorSlice(scanErrors, indexedFileProducerErrors)

scanResults := xrutils.NewAuditResults()
scanResults.XrayVersion = xrayVersion
scanResults.ScaResults = flatResults

// Wait for the Download of the AnalyzerManager to complete.
if err = errGroup.Wait(); err != nil {
err = errors.New("failed while trying to get Analyzer Manager: " + err.Error())
}

if err = xrutils.NewResultsWriter(scanResults).
SetOutputFormat(scanCmd.outputFormat).
SetIncludeVulnerabilities(scanCmd.includeVulnerabilities).
Expand Down Expand Up @@ -296,14 +327,14 @@ func (scanCmd *ScanCommand) CommandName() string {
return "xr_scan"
}

func (scanCmd *ScanCommand) prepareScanTasks(fileProducer, indexedFileProducer parallel.Runner, resultsArr [][]*ScanInfo, fileErrors, indexedFileErrors [][]formats.SimpleJsonError, fileCollectingErrorsQueue *clientutils.ErrorsQueue, xrayVersion string) {
func (scanCmd *ScanCommand) prepareScanTasks(fileProducer, indexedFileProducer parallel.Runner, entitledForJas bool, resultsArr [][]*ScanInfo, fileErrors, indexedFileErrors [][]formats.SimpleJsonError, fileCollectingErrorsQueue *clientutils.ErrorsQueue, xrayVersion string) {
go func() {
defer fileProducer.Done()
// Iterate over file-spec groups and produce indexing tasks.
// When encountering an error, log and move to next group.
specFiles := scanCmd.spec.Files
for i := range specFiles {
artifactHandlerFunc := scanCmd.createIndexerHandlerFunc(&specFiles[i], indexedFileProducer, resultsArr, fileErrors, indexedFileErrors, xrayVersion)
artifactHandlerFunc := scanCmd.createIndexerHandlerFunc(&specFiles[i], entitledForJas, indexedFileProducer, resultsArr, fileErrors, indexedFileErrors, xrayVersion)
taskHandler := getAddTaskToProducerFunc(fileProducer, artifactHandlerFunc)

err := collectFilesForIndexing(specFiles[i], taskHandler)
Expand All @@ -315,7 +346,7 @@ func (scanCmd *ScanCommand) prepareScanTasks(fileProducer, indexedFileProducer p
}()
}

func (scanCmd *ScanCommand) createIndexerHandlerFunc(file *spec.File, indexedFileProducer parallel.Runner, resultsArr [][]*ScanInfo, fileErrors, indexedFileErrors [][]formats.SimpleJsonError, xrayVersion string) FileContext {
func (scanCmd *ScanCommand) createIndexerHandlerFunc(file *spec.File, entitledForJas bool, indexedFileProducer parallel.Runner, resultsArr [][]*ScanInfo, fileErrors, indexedFileErrors [][]formats.SimpleJsonError, xrayVersion string) FileContext {
return func(filePath string) parallel.TaskFunc {
return func(threadId int) (err error) {
logMsgPrefix := clientutils.GetLogMsgPrefix(threadId, false)
Expand Down Expand Up @@ -360,12 +391,26 @@ func (scanCmd *ScanCommand) createIndexerHandlerFunc(file *spec.File, indexedFil
return err
}
scanResults, err := scangraph.RunScanGraphAndGetResults(scanGraphParams, xrayManager)

if err != nil {
log.Error(fmt.Sprintf("scanning '%s' failed with error: %s", graph.Id, err.Error()))
indexedFileErrors[threadId] = append(indexedFileErrors[threadId], formats.SimpleJsonError{FilePath: filePath, ErrorMessage: err.Error()})
return
}
resultsArr[threadId] = append(resultsArr[threadId], &ScanInfo{Target: filePath, Result: scanResults})

extendedScanResults := utils.ExtendedScanResults{}
if entitledForJas && scanCmd.commandSupportsJAS {
// Run Jas scans
workingDirs := []string{filePath}
err = runner.RunJasScannersAndSetResults(&extendedScanResults, []coreutils.Technology{coreutils.Technology(scanResults.ScannedPackageType)}, []services.ScanResponse{*scanResults}, depsListFromVulnerabilities(*scanResults), scanCmd.serverDetails, workingDirs, nil, false, "", applicability.ApplicabilityDockerScanScanType, secrets.SecretsScannerDockerScanType)

if err != nil {
log.Error(fmt.Sprintf("scanning '%s' failed with error: %s", graph.Id, err.Error()))
indexedFileErrors[threadId] = append(indexedFileErrors[threadId], formats.SimpleJsonError{FilePath: filePath, ErrorMessage: err.Error()})
return
}
}
resultsArr[threadId] = append(resultsArr[threadId], &ScanInfo{Target: filePath, Result: scanResults, ExtendedScanResults: &extendedScanResults})
return
}

Expand Down Expand Up @@ -469,6 +514,20 @@ func appendErrorSlice(scanErrors []formats.SimpleJsonError, errorsToAdd [][]form
return scanErrors
}

func depsListFromVulnerabilities(scanResult ...services.ScanResponse) (depsList []string) {
for _, result := range scanResult {
for _, vulnerability := range result.Vulnerabilities {
dependencies := maps.Keys(vulnerability.Components)
for _, dependency := range dependencies {
if !slices.Contains(depsList, dependency) {
depsList = append(depsList, dependency)
}
}
}
}
return
}

func ConditionalUploadDefaultScanFunc(serverDetails *config.ServerDetails, fileSpec *spec.SpecFiles, threads int, scanOutputFormat format.OutputFormat) error {
return NewScanCommand().SetServerDetails(serverDetails).SetSpec(fileSpec).SetThreads(threads).SetOutputFormat(scanOutputFormat).SetFail(true).SetPrintExtendedTable(false).Run()
}
1 change: 1 addition & 0 deletions formats/conversion.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ func ConvertToVulnerabilityScanTableRow(rows []VulnerabilityOrViolationRow) (tab
tableRows = append(tableRows, vulnerabilityScanTableRow{
severity: rows[i].Severity,
severityNumValue: rows[i].SeverityNumValue,
applicable: rows[i].Applicable,
impactedPackageName: rows[i].ImpactedDependencyName,
impactedPackageVersion: rows[i].ImpactedDependencyVersion,
ImpactedPackageType: rows[i].ImpactedDependencyType,
Expand Down
3 changes: 2 additions & 1 deletion formats/table.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,8 @@ type vulnerabilityTableRow struct {
}

type vulnerabilityScanTableRow struct {
severity string `col-name:"Severity"`
severity string `col-name:"Severity"`
applicable string `col-name:"Contextual\nAnalysis" omitempty:"true"`
// For sorting
severityNumValue int
directPackages []directPackagesTableRow `embed-table:"true"`
Expand Down
Loading
Loading