From 49b8080461bafbfbb4cb4f450da35e3fa27f459a Mon Sep 17 00:00:00 2001 From: Ralf Pannemans Date: Mon, 12 Feb 2024 08:56:40 +0100 Subject: [PATCH] feat(detectExecuteScan): Also scan images that are in the CPE (#4678) feat(detectExecuteScan): Also scan images that are in the cpe Signed-off-by: Ralf Pannemans Signed-off-by: Johannes Dillmann Signed-off-by: Pavel Busko Co-authored-by: Johannes Dillmann Co-authored-by: Pavel Busko --- cmd/detectExecuteScan.go | 120 ++++++++++++++++++++-- cmd/detectExecuteScan_generated.go | 80 +++++++++++++++ cmd/detectExecuteScan_test.go | 2 +- pkg/docker/docker.go | 3 +- resources/metadata/detectExecuteScan.yaml | 116 +++++++++++++++++++++ 5 files changed, 310 insertions(+), 11 deletions(-) diff --git a/cmd/detectExecuteScan.go b/cmd/detectExecuteScan.go index 5655582638..8cc5f45550 100644 --- a/cmd/detectExecuteScan.go +++ b/cmd/detectExecuteScan.go @@ -20,6 +20,7 @@ import ( "github.com/SAP/jenkins-library/pkg/log" "github.com/SAP/jenkins-library/pkg/maven" "github.com/SAP/jenkins-library/pkg/orchestrator" + "github.com/SAP/jenkins-library/pkg/piperenv" "github.com/SAP/jenkins-library/pkg/piperutils" "github.com/SAP/jenkins-library/pkg/reporting" "github.com/SAP/jenkins-library/pkg/telemetry" @@ -30,6 +31,8 @@ import ( "github.com/pkg/errors" ) +const NO_VERSION_SUFFIX = "" + type detectUtils interface { piperutils.FileUtils @@ -202,7 +205,7 @@ func runDetect(ctx context.Context, config detectExecuteScanOptions, utils detec blackduckSystem := newBlackduckSystem(config) args := []string{"./detect.sh"} - args, err = addDetectArgs(args, config, utils, blackduckSystem) + args, err = addDetectArgs(args, config, utils, blackduckSystem, NO_VERSION_SUFFIX, NO_VERSION_SUFFIX) if err != nil { return err } @@ -214,7 +217,18 @@ func runDetect(ctx context.Context, config detectExecuteScanOptions, utils detec utils.SetDir(".") utils.SetEnv(envs) - err = utils.RunShell("/bin/bash", script) + err = mapDetectError(utils.RunShell("/bin/bash", script), config, utils) + if config.ScanContainerDistro != "" { + imageError := mapDetectError(runDetectImages(ctx, config, utils, blackduckSystem, influx, blackduckSystem), config, utils) + if imageError != nil { + if err != nil { + err = errors.Wrapf(err, "error during scanning images: %q", imageError.Error()) + } else { + err = imageError + } + } + } + reportingErr := postScanChecksAndReporting(ctx, config, influx, utils, blackduckSystem) if reportingErr != nil { if strings.Contains(reportingErr.Error(), "License Policy Violations found") { @@ -227,6 +241,17 @@ func runDetect(ctx context.Context, config detectExecuteScanOptions, utils detec log.Entry().Warnf("Failed to generate reports: %v", reportingErr) } } + // create Toolrecord file + toolRecordFileName, toolRecordErr := createToolRecordDetect(utils, "./", config, blackduckSystem) + if toolRecordErr != nil { + // do not fail until the framework is well established + log.Entry().Warning("TR_DETECT: Failed to create toolrecord file "+toolRecordFileName, err) + } + + return err +} + +func mapDetectError(err error, config detectExecuteScanOptions, utils detectUtils) error { if err != nil { // Setting error category based on exit code mapErrorCategory(utils.GetExitCode()) @@ -238,15 +263,52 @@ func runDetect(ctx context.Context, config detectExecuteScanOptions, utils detec err = errors.Wrapf(err, exitCodeMapping(utils.GetExitCode())) } } - // create Toolrecord file - toolRecordFileName, toolRecordErr := createToolRecordDetect(utils, "./", config, blackduckSystem) - if toolRecordErr != nil { - // do not fail until the framework is well established - log.Entry().Warning("TR_DETECT: Failed to create toolrecord file "+toolRecordFileName, err) - } return err } +func runDetectImages(ctx context.Context, config detectExecuteScanOptions, utils detectUtils, sys *blackduckSystem, influx *detectExecuteScanInflux, blackduckSystem *blackduckSystem) error { + cpePath := filepath.Join(GeneralConfig.EnvRootPath, "commonPipelineEnvironment") + imagesRaw := piperenv.GetResourceParameter(cpePath, "container", "imageNameTags.json") + if imagesRaw == "" { + log.Entry().Debugf("No images found to be scanned") + return nil + } + + var err error + log.Entry().Infof("Scanning %d images", len(config.ImageNameTags)) + for _, image := range config.ImageNameTags { + // Download image to be scanned + log.Entry().Debugf("Scanning image: %q", image) + tarName := fmt.Sprintf("%s.tar", strings.Split(image, ":")[0]) + + options := containerSaveImageOptions{ + ContainerRegistryURL: config.RegistryURL, + ContainerImage: image, + ContainerRegistryPassword: config.RepositoryPassword, + ContainerRegistryUser: config.RepositoryUsername, + FilePath: tarName, + ImageFormat: "legacy", + } + containerSaveImage(options, &telemetry.CustomData{}) + + args := []string{"./detect.sh"} + args, err = addDetectArgsImages(args, config, utils, sys, tarName) + if err != nil { + return err + } + script := strings.Join(args, " ") + + err = utils.RunShell("/bin/bash", script) + err = mapDetectError(err, config, utils) + + if err != nil { + return err + } + } + + return nil +} + // Get proper error category func mapErrorCategory(exitCodeKey int) { switch exitCodeKey { @@ -331,8 +393,11 @@ func getDetectScript(config detectExecuteScanOptions, utils detectUtils) error { return nil } -func addDetectArgs(args []string, config detectExecuteScanOptions, utils detectUtils, sys *blackduckSystem) ([]string, error) { +func addDetectArgs(args []string, config detectExecuteScanOptions, utils detectUtils, sys *blackduckSystem, versionSuffix, locationSuffix string) ([]string, error) { detectVersionName := getVersionName(config) + if versionSuffix != NO_VERSION_SUFFIX { + detectVersionName = fmt.Sprintf("%s-%s", detectVersionName, versionSuffix) + } // Split on spaces, the scanPropeties, so that each property is available as a single string // instead of all properties being part of a single string config.ScanProperties = piperutils.SplitAndTrim(config.ScanProperties, " ") @@ -393,6 +458,10 @@ func addDetectArgs(args []string, config detectExecuteScanOptions, utils detectU codelocation := config.CodeLocation if len(codelocation) == 0 && len(config.ProjectName) > 0 { codelocation = fmt.Sprintf("%v/%v", config.ProjectName, detectVersionName) + + if locationSuffix != "" { + codelocation = fmt.Sprintf("%v-%v", codelocation, locationSuffix) + } } args = append(args, fmt.Sprintf("\"--detect.project.name=%v\"", config.ProjectName)) @@ -467,6 +536,39 @@ func addDetectArgs(args []string, config detectExecuteScanOptions, utils detectU return args, nil } +func addDetectArgsImages(args []string, config detectExecuteScanOptions, utils detectUtils, sys *blackduckSystem, imageTar string) ([]string, error) { + // suffix := strings.Split(imageTar, ".")[0] + // In order to preserve source scan result + config.Unmap = false + args, err := addDetectArgs(args, config, utils, sys, NO_VERSION_SUFFIX, fmt.Sprintf("image-%s", strings.Split(imageTar, ".")[0])) + if err != nil { + return []string{}, err + } + + args = append(args, fmt.Sprintf("--detect.docker.tar=./%s", imageTar)) + args = append(args, "--detect.target.type=IMAGE") + // https://community.synopsys.com/s/article/Docker-image-scanning-CLI-examples-and-some-Q-As + args = append(args, "--detect.tools.excluded=DETECTOR") + args = append(args, "--detect.docker.passthrough.shared.dir.path.local=/opt/blackduck/blackduck-imageinspector/shared/") + args = append(args, "--detect.docker.passthrough.shared.dir.path.imageinspector=/opt/blackduck/blackduck-imageinspector/shared") + args = append(args, fmt.Sprintf("--detect.docker.passthrough.imageinspector.service.distro.default=%s", config.ScanContainerDistro)) + args = append(args, "--detect.docker.passthrough.imageinspector.service.start=false") + args = append(args, "--detect.docker.passthrough.output.include.squashedimage=false") + + switch config.ScanContainerDistro { + case "ubuntu": + args = append(args, "--detect.docker.passthrough.imageinspector.service.url=http://localhost:8082") + case "centos": + args = append(args, "--detect.docker.passthrough.imageinspector.service.url=http://localhost:8081") + case "alpine": + args = append(args, "--detect.docker.passthrough.imageinspector.service.url=http://localhost:8080") + default: + return nil, fmt.Errorf("unknown container distro %q", config.ScanContainerDistro) + } + + return args, nil +} + func getVersionName(config detectExecuteScanOptions) string { detectVersionName := config.CustomScanVersion if len(detectVersionName) > 0 { diff --git a/cmd/detectExecuteScan_generated.go b/cmd/detectExecuteScan_generated.go index 128b5ca9d1..5c07d6027a 100644 --- a/cmd/detectExecuteScan_generated.go +++ b/cmd/detectExecuteScan_generated.go @@ -64,6 +64,11 @@ type detectExecuteScanOptions struct { NpmArguments []string `json:"npmArguments,omitempty"` PrivateModules string `json:"privateModules,omitempty"` PrivateModulesGitToken string `json:"privateModulesGitToken,omitempty"` + ScanContainerDistro string `json:"scanContainerDistro,omitempty" validate:"possible-values=ubuntu centos alpine"` + ImageNameTags []string `json:"imageNameTags,omitempty" validate:"required_if=ScanContainerDistro ubuntu ScanContainerDistro centos ScanContainerDistro alpine"` + RegistryURL string `json:"registryUrl,omitempty" validate:"required_if=ScanContainerDistro ubuntu ScanContainerDistro centos ScanContainerDistro alpine"` + RepositoryUsername string `json:"repositoryUsername,omitempty" validate:"required_if=ScanContainerDistro ubuntu ScanContainerDistro centos ScanContainerDistro alpine"` + RepositoryPassword string `json:"repositoryPassword,omitempty" validate:"required_if=ScanContainerDistro ubuntu ScanContainerDistro centos ScanContainerDistro alpine"` } type detectExecuteScanInflux struct { @@ -310,6 +315,11 @@ func addDetectExecuteScanFlags(cmd *cobra.Command, stepConfig *detectExecuteScan cmd.Flags().StringSliceVar(&stepConfig.NpmArguments, "npmArguments", []string{}, "List of additional arguments that Detect will add at then end of the npm ls command line when Detect executes the NPM CLI Detector on an NPM project.") cmd.Flags().StringVar(&stepConfig.PrivateModules, "privateModules", os.Getenv("PIPER_privateModules"), "Tells go which modules shall be considered to be private (by setting [GOPRIVATE](https://pkg.go.dev/cmd/go#hdr-Configuration_for_downloading_non_public_code)).") cmd.Flags().StringVar(&stepConfig.PrivateModulesGitToken, "privateModulesGitToken", os.Getenv("PIPER_privateModulesGitToken"), "GitHub personal access token as per https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line.") + cmd.Flags().StringVar(&stepConfig.ScanContainerDistro, "scanContainerDistro", os.Getenv("PIPER_scanContainerDistro"), "To also scan your images in the CPE, choose the distro") + cmd.Flags().StringSliceVar(&stepConfig.ImageNameTags, "imageNameTags", []string{}, "Images to be scanned (typically filled by CPE)") + cmd.Flags().StringVar(&stepConfig.RegistryURL, "registryUrl", os.Getenv("PIPER_registryUrl"), "Used accessing for the images to be scanned (typically filled by CPE)") + cmd.Flags().StringVar(&stepConfig.RepositoryUsername, "repositoryUsername", os.Getenv("PIPER_repositoryUsername"), "Used accessing for the images to be scanned (typically filled by CPE)") + cmd.Flags().StringVar(&stepConfig.RepositoryPassword, "repositoryPassword", os.Getenv("PIPER_repositoryPassword"), "Used accessing for the images to be scanned (typically filled by CPE)") cmd.MarkFlagRequired("token") cmd.MarkFlagRequired("projectName") @@ -773,11 +783,81 @@ func detectExecuteScanMetadata() config.StepData { Aliases: []config.Alias{}, Default: os.Getenv("PIPER_privateModulesGitToken"), }, + { + Name: "scanContainerDistro", + ResourceRef: []config.ResourceReference{}, + Scope: []string{"PARAMETERS", "STAGES", "STEPS"}, + Type: "string", + Mandatory: false, + Aliases: []config.Alias{}, + Default: os.Getenv("PIPER_scanContainerDistro"), + }, + { + Name: "imageNameTags", + ResourceRef: []config.ResourceReference{ + { + Name: "commonPipelineEnvironment", + Param: "container/imageNameTags", + }, + }, + Scope: []string{"STEPS", "STAGES", "PARAMETERS"}, + Type: "[]string", + Mandatory: false, + Aliases: []config.Alias{}, + Default: []string{}, + }, + { + Name: "registryUrl", + ResourceRef: []config.ResourceReference{ + { + Name: "commonPipelineEnvironment", + Param: "container/registryUrl", + }, + }, + Scope: []string{"STEPS", "STAGES", "PARAMETERS"}, + Type: "string", + Mandatory: false, + Aliases: []config.Alias{}, + Default: os.Getenv("PIPER_registryUrl"), + }, + { + Name: "repositoryUsername", + ResourceRef: []config.ResourceReference{ + { + Name: "commonPipelineEnvironment", + Param: "container/repositoryUsername", + }, + }, + Scope: []string{"STEPS", "STAGES", "PARAMETERS"}, + Type: "string", + Mandatory: false, + Aliases: []config.Alias{}, + Default: os.Getenv("PIPER_repositoryUsername"), + }, + { + Name: "repositoryPassword", + ResourceRef: []config.ResourceReference{ + { + Name: "commonPipelineEnvironment", + Param: "container/repositoryPassword", + }, + }, + Scope: []string{"STEPS", "STAGES", "PARAMETERS"}, + Type: "string", + Mandatory: false, + Aliases: []config.Alias{}, + Default: os.Getenv("PIPER_repositoryPassword"), + }, }, }, Containers: []config.Container{ {Name: "openjdk", Image: "openjdk:11", WorkingDir: "/root", Options: []config.Option{{Name: "-u", Value: "0"}}}, }, + Sidecars: []config.Container{ + {Name: "inspector-ubuntu", Image: "blackducksoftware/blackduck-imageinspector-ubuntu:5.1.0", Conditions: []config.Condition{{ConditionRef: "strings-equal", Params: []config.Param{{Name: "scanContainerDistro", Value: "ubuntu"}}}}}, + {Name: "inspector-alpine", Image: "blackducksoftware/blackduck-imageinspector-alpine:5.1.0", Conditions: []config.Condition{{ConditionRef: "strings-equal", Params: []config.Param{{Name: "scanContainerDistro", Value: "alpine"}}}}}, + {Name: "inspector-centos", Image: "blackducksoftware/blackduck-imageinspector-centos:5.1.0", Conditions: []config.Condition{{ConditionRef: "strings-equal", Params: []config.Param{{Name: "scanContainerDistro", Value: "centos"}}}}}, + }, Outputs: config.StepOutputs{ Resources: []config.StepResources{ { diff --git a/cmd/detectExecuteScan_test.go b/cmd/detectExecuteScan_test.go index 0fbcee1326..d76bbfb406 100644 --- a/cmd/detectExecuteScan_test.go +++ b/cmd/detectExecuteScan_test.go @@ -807,7 +807,7 @@ func TestAddDetectArgs(t *testing.T) { config := detectExecuteScanOptions{Token: "token", ServerURL: "https://my.blackduck.system", ProjectName: v.options.ProjectName, Version: v.options.Version, CustomScanVersion: v.options.CustomScanVersion} sys := newBlackduckMockSystem(config) - got, err := addDetectArgs(v.args, v.options, newDetectTestUtilsBundle(v.isPullRequest), &sys) + got, err := addDetectArgs(v.args, v.options, newDetectTestUtilsBundle(v.isPullRequest), &sys, NO_VERSION_SUFFIX, "") assert.NoError(t, err) assert.Equal(t, v.expected, got) }) diff --git a/pkg/docker/docker.go b/pkg/docker/docker.go index d0619b7dd9..d4f757e00c 100644 --- a/pkg/docker/docker.go +++ b/pkg/docker/docker.go @@ -244,7 +244,8 @@ func (c *Client) DownloadImage(imageSource, targetFile string) (v1.Image, error) craneCmd := cranecmd.NewCmdPull(&noOpts) craneCmd.SetOut(log.Writer()) craneCmd.SetErr(log.Writer()) - craneCmd.SetArgs([]string{imageRef.Name(), tmpFile.Name(), "--format=" + c.imageFormat}) + args := []string{imageRef.Name(), tmpFile.Name(), "--format=" + c.imageFormat} + craneCmd.SetArgs(args) if err := craneCmd.Execute(); err != nil { defer os.Remove(tmpFile.Name()) diff --git a/resources/metadata/detectExecuteScan.yaml b/resources/metadata/detectExecuteScan.yaml index d03dd4b8a7..2110312a2e 100644 --- a/resources/metadata/detectExecuteScan.yaml +++ b/resources/metadata/detectExecuteScan.yaml @@ -511,6 +511,85 @@ spec: - type: vaultSecret name: golangPrivateModulesGitTokenVaultSecret default: golang + - name: scanContainerDistro + description: To also scan your images in the CPE, choose the distro + type: "string" + scope: + - PARAMETERS + - STAGES + - STEPS + possibleValues: + - ubuntu + - centos + - alpine + - name: imageNameTags + type: "[]string" + mandatoryIf: + - name: scanContainerDistro + value: ubuntu + - name: scanContainerDistro + value: centos + - name: scanContainerDistro + value: alpine + description: Images to be scanned (typically filled by CPE) + scope: + - STEPS + - STAGES + - PARAMETERS + resourceRef: + - name: commonPipelineEnvironment + param: container/imageNameTags + - name: registryUrl + type: string + mandatoryIf: + - name: scanContainerDistro + value: ubuntu + - name: scanContainerDistro + value: centos + - name: scanContainerDistro + value: alpine + description: Used accessing for the images to be scanned (typically filled by CPE) + scope: + - STEPS + - STAGES + - PARAMETERS + resourceRef: + - name: commonPipelineEnvironment + param: container/registryUrl + - name: repositoryUsername + type: string + mandatoryIf: + - name: scanContainerDistro + value: ubuntu + - name: scanContainerDistro + value: centos + - name: scanContainerDistro + value: alpine + description: Used accessing for the images to be scanned (typically filled by CPE) + scope: + - STEPS + - STAGES + - PARAMETERS + resourceRef: + - name: commonPipelineEnvironment + param: container/repositoryUsername + - name: repositoryPassword + type: string + mandatoryIf: + - name: scanContainerDistro + value: ubuntu + - name: scanContainerDistro + value: centos + - name: scanContainerDistro + value: alpine + description: Used accessing for the images to be scanned (typically filled by CPE) + scope: + - STEPS + - STAGES + - PARAMETERS + resourceRef: + - name: commonPipelineEnvironment + param: container/repositoryPassword outputs: resources: - name: influx @@ -562,3 +641,40 @@ spec: options: - name: -u value: "0" + volumeMounts: + - mountPath: /opt/blackduck/blackduck-imageinspector/shared + name: volume + sidecars: + - name: inspector-ubuntu + image: blackducksoftware/blackduck-imageinspector-ubuntu:5.1.0 + command: [''] + volumeMounts: + - mountPath: /opt/blackduck/blackduck-imageinspector/shared + name: volume + conditions: + - conditionRef: strings-equal + params: + - name: scanContainerDistro + value: ubuntu + - name: inspector-alpine + image: blackducksoftware/blackduck-imageinspector-alpine:5.1.0 + command: [''] + volumeMounts: + - mountPath: /opt/blackduck/blackduck-imageinspector/shared + name: volume + conditions: + - conditionRef: strings-equal + params: + - name: scanContainerDistro + value: alpine + - name: inspector-centos + image: blackducksoftware/blackduck-imageinspector-centos:5.1.0 + command: [''] + volumeMounts: + - mountPath: /opt/blackduck/blackduck-imageinspector/shared + name: volume + conditions: + - conditionRef: strings-equal + params: + - name: scanContainerDistro + value: centos