From bf59a28abaca8b79efeb24409e4dad55f5e21872 Mon Sep 17 00:00:00 2001 From: Daria Kuznetsova Date: Tue, 2 Apr 2024 06:48:17 +0200 Subject: [PATCH] feat(codeqlExecuteScan): added open configs for codeql database creation and analysis (#4869) Co-authored-by: sumeet patil --- cmd/codeqlExecuteScan.go | 173 +++++---- cmd/codeqlExecuteScan_generated.go | 22 ++ cmd/codeqlExecuteScan_test.go | 405 ++++++++++++++++++++-- pkg/codeql/flags.go | 128 +++++++ pkg/codeql/flags_test.go | 329 ++++++++++++++++++ resources/metadata/codeqlExecuteScan.yaml | 24 ++ 6 files changed, 982 insertions(+), 99 deletions(-) create mode 100644 pkg/codeql/flags.go create mode 100644 pkg/codeql/flags_test.go diff --git a/cmd/codeqlExecuteScan.go b/cmd/codeqlExecuteScan.go index 16e33a44a6..f54e1ad8e6 100644 --- a/cmd/codeqlExecuteScan.go +++ b/cmd/codeqlExecuteScan.go @@ -67,7 +67,7 @@ func codeqlExecuteScan(config codeqlExecuteScanOptions, telemetryData *telemetry influx.step_data.fields.codeql = true } -func codeqlQuery(cmd []string, codeqlQuery string) []string { +func appendCodeqlQuery(cmd []string, codeqlQuery string) []string { if len(codeqlQuery) > 0 { cmd = append(cmd, codeqlQuery) } @@ -189,27 +189,7 @@ func getToken(config *codeqlExecuteScanOptions) (bool, string) { } func uploadResults(config *codeqlExecuteScanOptions, repoInfo codeql.RepoInfo, token string, utils codeqlExecuteScanUtils) (string, error) { - cmd := []string{"github", "upload-results", "--sarif=" + filepath.Join(config.ModulePath, "target", "codeqlReport.sarif")} - - if config.GithubToken != "" { - cmd = append(cmd, "-a="+token) - } - - if repoInfo.CommitId != "" { - cmd = append(cmd, "--commit="+repoInfo.CommitId) - } - - if repoInfo.ServerUrl != "" { - cmd = append(cmd, "--github-url="+repoInfo.ServerUrl) - } - - if repoInfo.Repo != "" { - cmd = append(cmd, "--repository="+(repoInfo.Owner+"/"+repoInfo.Repo)) - } - - if repoInfo.Ref != "" { - cmd = append(cmd, "--ref="+repoInfo.Ref) - } + cmd := prepareCmdForUploadResults(config, &repoInfo, token) //if no git params are passed(commitId, reference, serverUrl, repository), then codeql tries to auto populate it based on git information of the checkout repository. //It also depends on the orchestrator. Some orchestrator keep git information and some not. @@ -275,29 +255,12 @@ func runCodeqlExecuteScan(config *codeqlExecuteScanOptions, telemetryData *telem } var reports []piperutils.Path - cmd := []string{"database", "create", config.Database, "--overwrite", "--source-root", ".", "--working-dir", config.ModulePath} - - language := getLangFromBuildTool(config.BuildTool) - if len(language) == 0 && len(config.Language) == 0 { - if config.BuildTool == "custom" { - return reports, fmt.Errorf("as the buildTool is custom. please specify the language parameter") - } else { - return reports, fmt.Errorf("the step could not recognize the specified buildTool %s. please specify valid buildtool", config.BuildTool) - } - } - if len(language) > 0 { - cmd = append(cmd, "--language="+language) - } else { - cmd = append(cmd, "--language="+config.Language) - } - - cmd = append(cmd, getRamAndThreadsFromConfig(config)...) - - if len(config.BuildCommand) > 0 { - buildCmd := config.BuildCommand - buildCmd = buildCmd + getMavenSettings(config, utils) - cmd = append(cmd, "--command="+buildCmd) + dbCreateCustomFlags := codeql.ParseCustomFlags(config.DatabaseCreateFlags) + cmd, err := prepareCmdForDatabaseCreate(dbCreateCustomFlags, config, utils) + if err != nil { + log.Entry().WithError(err).Error("failed to prepare command for codeql database create") + return reports, err } err = execute(utils, cmd, GeneralConfig.Verbose) @@ -311,28 +274,29 @@ func runCodeqlExecuteScan(config *codeqlExecuteScanOptions, telemetryData *telem return reports, fmt.Errorf("failed to create directory: %w", err) } - cmd = nil - cmd = append(cmd, "database", "analyze", "--format=sarif-latest", fmt.Sprintf("--output=%v", filepath.Join(config.ModulePath, "target", "codeqlReport.sarif")), config.Database) - cmd = append(cmd, getRamAndThreadsFromConfig(config)...) - cmd = codeqlQuery(cmd, config.QuerySuite) + dbAnalyzeCustomFlags := codeql.ParseCustomFlags(config.DatabaseAnalyzeFlags) + cmd, err = prepareCmdForDatabaseAnalyze(dbAnalyzeCustomFlags, config, "sarif-latest", "codeqlReport.sarif") + if err != nil { + log.Entry().WithError(err).Error("failed to prepare command for codeql database analyze format=sarif-latest") + return reports, err + } err = execute(utils, cmd, GeneralConfig.Verbose) if err != nil { log.Entry().Error("failed running command codeql database analyze for sarif generation") return reports, err } - reports = append(reports, piperutils.Path{Target: filepath.Join(config.ModulePath, "target", "codeqlReport.sarif")}) - cmd = nil - cmd = append(cmd, "database", "analyze", "--format=csv", fmt.Sprintf("--output=%v", filepath.Join(config.ModulePath, "target", "codeqlReport.csv")), config.Database) - cmd = append(cmd, getRamAndThreadsFromConfig(config)...) - cmd = codeqlQuery(cmd, config.QuerySuite) + cmd, err = prepareCmdForDatabaseAnalyze(dbAnalyzeCustomFlags, config, "csv", "codeqlReport.csv") + if err != nil { + log.Entry().WithError(err).Error("failed to prepare command for codeql database analyze format=csv") + return reports, err + } err = execute(utils, cmd, GeneralConfig.Verbose) if err != nil { log.Entry().Error("failed running command codeql database analyze for csv generation") return reports, err } - reports = append(reports, piperutils.Path{Target: filepath.Join(config.ModulePath, "target", "codeqlReport.csv")}) repoInfo, err := initGitInfo(config) @@ -427,6 +391,80 @@ func runCodeqlExecuteScan(config *codeqlExecuteScanOptions, telemetryData *telem return reports, nil } +func prepareCmdForDatabaseCreate(customFlags map[string]string, config *codeqlExecuteScanOptions, utils codeqlExecuteScanUtils) ([]string, error) { + cmd := []string{"database", "create", config.Database} + cmd = codeql.AppendFlagIfNotSetByUser(cmd, []string{"--overwrite", "--no-overwrite"}, []string{"--overwrite"}, customFlags) + cmd = codeql.AppendFlagIfNotSetByUser(cmd, []string{"--source-root", "-s"}, []string{"--source-root", "."}, customFlags) + cmd = codeql.AppendFlagIfNotSetByUser(cmd, []string{"--working-dir"}, []string{"--working-dir", config.ModulePath}, customFlags) + + if !codeql.IsFlagSetByUser(customFlags, []string{"--language", "-l"}) { + language := getLangFromBuildTool(config.BuildTool) + if len(language) == 0 && len(config.Language) == 0 { + if config.BuildTool == "custom" { + return nil, fmt.Errorf("as the buildTool is custom. please specify the language parameter") + } else { + return nil, fmt.Errorf("the step could not recognize the specified buildTool %s. please specify valid buildtool", config.BuildTool) + } + } + if len(language) > 0 { + cmd = append(cmd, "--language="+language) + } else { + cmd = append(cmd, "--language="+config.Language) + } + } + + cmd = codeql.AppendThreadsAndRam(cmd, config.Threads, config.Ram, customFlags) + + if len(config.BuildCommand) > 0 && !codeql.IsFlagSetByUser(customFlags, []string{"--command", "-c"}) { + buildCmd := config.BuildCommand + buildCmd = buildCmd + getMavenSettings(buildCmd, config, utils) + cmd = append(cmd, "--command="+buildCmd) + } + + if codeql.IsFlagSetByUser(customFlags, []string{"--command", "-c"}) { + updateCmdFlag(config, customFlags, utils) + } + cmd = codeql.AppendCustomFlags(cmd, customFlags) + + return cmd, nil +} + +func prepareCmdForDatabaseAnalyze(customFlags map[string]string, config *codeqlExecuteScanOptions, format, reportName string) ([]string, error) { + cmd := []string{"database", "analyze", "--format=" + format, fmt.Sprintf("--output=%v", filepath.Join(config.ModulePath, "target", reportName)), config.Database} + cmd = codeql.AppendThreadsAndRam(cmd, config.Threads, config.Ram, customFlags) + cmd = codeql.AppendCustomFlags(cmd, customFlags) + cmd = appendCodeqlQuery(cmd, config.QuerySuite) + return cmd, nil +} + +func prepareCmdForUploadResults(config *codeqlExecuteScanOptions, repoInfo *codeql.RepoInfo, token string) []string { + cmd := []string{"github", "upload-results", "--sarif=" + filepath.Join(config.ModulePath, "target", "codeqlReport.sarif")} + + //if no git params are passed(commitId, reference, serverUrl, repository), then codeql tries to auto populate it based on git information of the checkout repository. + //It also depends on the orchestrator. Some orchestrator keep git information and some not. + + if token != "" { + cmd = append(cmd, "-a="+token) + } + + if repoInfo.CommitId != "" { + cmd = append(cmd, "--commit="+repoInfo.CommitId) + } + + if repoInfo.ServerUrl != "" { + cmd = append(cmd, "--github-url="+repoInfo.ServerUrl) + } + + if repoInfo.Repo != "" && repoInfo.Owner != "" { + cmd = append(cmd, "--repository="+(repoInfo.Owner+"/"+repoInfo.Repo)) + } + + if repoInfo.Ref != "" { + cmd = append(cmd, "--ref="+repoInfo.Ref) + } + return cmd +} + func addDataToInfluxDB(repoUrl, repoRef, repoScanUrl, querySuite string, scanResults []codeql.CodeqlFindings, influx *codeqlExecuteScanInflux) { influx.codeql_data.fields.repositoryURL = repoUrl influx.codeql_data.fields.repositoryReferenceURL = repoRef @@ -445,20 +483,9 @@ func addDataToInfluxDB(repoUrl, repoRef, repoScanUrl, querySuite string, scanRes } } -func getRamAndThreadsFromConfig(config *codeqlExecuteScanOptions) []string { - params := make([]string, 0, 2) - if len(config.Threads) > 0 { - params = append(params, "--threads="+config.Threads) - } - if len(config.Ram) > 0 { - params = append(params, "--ram="+config.Ram) - } - return params -} - -func getMavenSettings(config *codeqlExecuteScanOptions, utils codeqlExecuteScanUtils) string { +func getMavenSettings(buildCmd string, config *codeqlExecuteScanOptions, utils codeqlExecuteScanUtils) string { params := "" - if len(config.BuildCommand) > 0 && config.BuildTool == "maven" && !strings.Contains(config.BuildCommand, "--global-settings") && !strings.Contains(config.BuildCommand, "--settings") { + if len(buildCmd) > 0 && config.BuildTool == "maven" && !strings.Contains(buildCmd, "--global-settings") && !strings.Contains(buildCmd, "--settings") { mvnParams, err := maven.DownloadAndGetMavenParameters(config.GlobalSettingsFile, config.ProjectSettingsFile, utils) if err != nil { log.Entry().Error("failed to download and get maven parameters: ", err) @@ -470,3 +497,15 @@ func getMavenSettings(config *codeqlExecuteScanOptions, utils codeqlExecuteScanU } return params } + +func updateCmdFlag(config *codeqlExecuteScanOptions, customFlags map[string]string, utils codeqlExecuteScanUtils) { + var buildCmd string + if customFlags["--command"] != "" { + buildCmd = customFlags["--command"] + } else { + buildCmd = customFlags["-c"] + } + buildCmd += getMavenSettings(buildCmd, config, utils) + customFlags["--command"] = buildCmd + delete(customFlags, "-c") +} diff --git a/cmd/codeqlExecuteScan_generated.go b/cmd/codeqlExecuteScan_generated.go index 36da7a51d9..b7cb31e10c 100644 --- a/cmd/codeqlExecuteScan_generated.go +++ b/cmd/codeqlExecuteScan_generated.go @@ -43,6 +43,8 @@ type codeqlExecuteScanOptions struct { CheckForCompliance bool `json:"checkForCompliance,omitempty"` ProjectSettingsFile string `json:"projectSettingsFile,omitempty"` GlobalSettingsFile string `json:"globalSettingsFile,omitempty"` + DatabaseCreateFlags string `json:"databaseCreateFlags,omitempty"` + DatabaseAnalyzeFlags string `json:"databaseAnalyzeFlags,omitempty"` } type codeqlExecuteScanInflux struct { @@ -267,6 +269,8 @@ func addCodeqlExecuteScanFlags(cmd *cobra.Command, stepConfig *codeqlExecuteScan cmd.Flags().BoolVar(&stepConfig.CheckForCompliance, "checkForCompliance", false, "If set to true, the piper step checks for compliance based on vulnerability threadholds. Example - If total vulnerabilites are 10 and vulnerabilityThresholdTotal is set as 0, then the steps throws an compliance error.") cmd.Flags().StringVar(&stepConfig.ProjectSettingsFile, "projectSettingsFile", os.Getenv("PIPER_projectSettingsFile"), "Path to the mvn settings file that should be used as project settings file.") cmd.Flags().StringVar(&stepConfig.GlobalSettingsFile, "globalSettingsFile", os.Getenv("PIPER_globalSettingsFile"), "Path to the mvn settings file that should be used as global settings file.") + cmd.Flags().StringVar(&stepConfig.DatabaseCreateFlags, "databaseCreateFlags", os.Getenv("PIPER_databaseCreateFlags"), "A space-separated string of flags for the 'codeql database create' command.") + cmd.Flags().StringVar(&stepConfig.DatabaseAnalyzeFlags, "databaseAnalyzeFlags", os.Getenv("PIPER_databaseAnalyzeFlags"), "A space-separated string of flags for the 'codeql database analyze' command.") cmd.MarkFlagRequired("buildTool") } @@ -505,6 +509,24 @@ func codeqlExecuteScanMetadata() config.StepData { Aliases: []config.Alias{{Name: "maven/globalSettingsFile"}}, Default: os.Getenv("PIPER_globalSettingsFile"), }, + { + Name: "databaseCreateFlags", + ResourceRef: []config.ResourceReference{}, + Scope: []string{"STEPS", "STAGES", "PARAMETERS"}, + Type: "string", + Mandatory: false, + Aliases: []config.Alias{}, + Default: os.Getenv("PIPER_databaseCreateFlags"), + }, + { + Name: "databaseAnalyzeFlags", + ResourceRef: []config.ResourceReference{}, + Scope: []string{"STEPS", "STAGES", "PARAMETERS"}, + Type: "string", + Mandatory: false, + Aliases: []config.Alias{}, + Default: os.Getenv("PIPER_databaseAnalyzeFlags"), + }, }, }, Containers: []config.Container{ diff --git a/cmd/codeqlExecuteScan_test.go b/cmd/codeqlExecuteScan_test.go index 3a638ecce3..fbf8e25fb5 100644 --- a/cmd/codeqlExecuteScan_test.go +++ b/cmd/codeqlExecuteScan_test.go @@ -4,6 +4,7 @@ package cmd import ( + "strings" "testing" "time" @@ -308,101 +309,179 @@ func TestGetMavenSettings(t *testing.T) { t.Parallel() t.Run("No maven", func(t *testing.T) { config := codeqlExecuteScanOptions{BuildTool: "npm"} - params := getMavenSettings(&config, newCodeqlExecuteScanTestsUtils()) + buildCmd := "mvn clean install" + params := getMavenSettings(buildCmd, &config, newCodeqlExecuteScanTestsUtils()) assert.Equal(t, "", params) }) t.Run("No build command", func(t *testing.T) { config := codeqlExecuteScanOptions{BuildTool: "maven"} - params := getMavenSettings(&config, newCodeqlExecuteScanTestsUtils()) + params := getMavenSettings("", &config, newCodeqlExecuteScanTestsUtils()) assert.Equal(t, "", params) }) t.Run("Project Settings file", func(t *testing.T) { - config := codeqlExecuteScanOptions{BuildTool: "maven", BuildCommand: "mvn clean install", ProjectSettingsFile: "test.xml"} - params := getMavenSettings(&config, newCodeqlExecuteScanTestsUtils()) + config := codeqlExecuteScanOptions{BuildTool: "maven", ProjectSettingsFile: "test.xml"} + buildCmd := "mvn clean install" + params := getMavenSettings(buildCmd, &config, newCodeqlExecuteScanTestsUtils()) assert.Equal(t, " --settings=test.xml", params) }) - t.Run("Skip Project Settings file incase already used", func(t *testing.T) { - config := codeqlExecuteScanOptions{BuildTool: "maven", BuildCommand: "mvn clean install --settings=project.xml", ProjectSettingsFile: "test.xml"} - params := getMavenSettings(&config, newCodeqlExecuteScanTestsUtils()) + t.Run("Skip Project Settings file in case already used", func(t *testing.T) { + config := codeqlExecuteScanOptions{BuildTool: "maven", ProjectSettingsFile: "test.xml"} + buildCmd := "mvn clean install --settings=project.xml" + params := getMavenSettings(buildCmd, &config, newCodeqlExecuteScanTestsUtils()) assert.Equal(t, "", params) }) t.Run("Global Settings file", func(t *testing.T) { - config := codeqlExecuteScanOptions{BuildTool: "maven", BuildCommand: "mvn clean install", GlobalSettingsFile: "gloabl.xml"} - params := getMavenSettings(&config, newCodeqlExecuteScanTestsUtils()) - assert.Equal(t, " --global-settings=gloabl.xml", params) + config := codeqlExecuteScanOptions{BuildTool: "maven", GlobalSettingsFile: "global.xml"} + buildCmd := "mvn clean install" + params := getMavenSettings(buildCmd, &config, newCodeqlExecuteScanTestsUtils()) + assert.Equal(t, " --global-settings=global.xml", params) }) t.Run("Project and Global Settings file", func(t *testing.T) { - config := codeqlExecuteScanOptions{BuildTool: "maven", BuildCommand: "mvn clean install", ProjectSettingsFile: "test.xml", GlobalSettingsFile: "global.xml"} - params := getMavenSettings(&config, newCodeqlExecuteScanTestsUtils()) + config := codeqlExecuteScanOptions{BuildTool: "maven", ProjectSettingsFile: "test.xml", GlobalSettingsFile: "global.xml"} + buildCmd := "mvn clean install" + params := getMavenSettings(buildCmd, &config, newCodeqlExecuteScanTestsUtils()) assert.Equal(t, " --global-settings=global.xml --settings=test.xml", params) }) t.Run("ProjectSettingsFile https url", func(t *testing.T) { - config := codeqlExecuteScanOptions{BuildTool: "maven", BuildCommand: "mvn clean install", ProjectSettingsFile: "https://jenkins-sap-test.com/test.xml"} - params := getMavenSettings(&config, newCodeqlExecuteScanTestsUtils()) + config := codeqlExecuteScanOptions{BuildTool: "maven", ProjectSettingsFile: "https://jenkins-sap-test.com/test.xml"} + buildCmd := "mvn clean install" + params := getMavenSettings(buildCmd, &config, newCodeqlExecuteScanTestsUtils()) assert.Equal(t, " --settings=.pipeline/mavenProjectSettings.xml", params) }) t.Run("ProjectSettingsFile http url", func(t *testing.T) { - config := codeqlExecuteScanOptions{BuildTool: "maven", BuildCommand: "mvn clean install", ProjectSettingsFile: "http://jenkins-sap-test.com/test.xml"} - params := getMavenSettings(&config, newCodeqlExecuteScanTestsUtils()) + config := codeqlExecuteScanOptions{BuildTool: "maven", ProjectSettingsFile: "http://jenkins-sap-test.com/test.xml"} + buildCmd := "mvn clean install" + params := getMavenSettings(buildCmd, &config, newCodeqlExecuteScanTestsUtils()) assert.Equal(t, " --settings=.pipeline/mavenProjectSettings.xml", params) }) t.Run("GlobalSettingsFile https url", func(t *testing.T) { - config := codeqlExecuteScanOptions{BuildTool: "maven", BuildCommand: "mvn clean install", GlobalSettingsFile: "https://jenkins-sap-test.com/test.xml"} - params := getMavenSettings(&config, newCodeqlExecuteScanTestsUtils()) + config := codeqlExecuteScanOptions{BuildTool: "maven", GlobalSettingsFile: "https://jenkins-sap-test.com/test.xml"} + buildCmd := "mvn clean install" + params := getMavenSettings(buildCmd, &config, newCodeqlExecuteScanTestsUtils()) assert.Equal(t, " --global-settings=.pipeline/mavenGlobalSettings.xml", params) }) t.Run("GlobalSettingsFile http url", func(t *testing.T) { - config := codeqlExecuteScanOptions{BuildTool: "maven", BuildCommand: "mvn clean install", GlobalSettingsFile: "http://jenkins-sap-test.com/test.xml"} - params := getMavenSettings(&config, newCodeqlExecuteScanTestsUtils()) + config := codeqlExecuteScanOptions{BuildTool: "maven", GlobalSettingsFile: "http://jenkins-sap-test.com/test.xml"} + buildCmd := "mvn clean install" + params := getMavenSettings(buildCmd, &config, newCodeqlExecuteScanTestsUtils()) assert.Equal(t, " --global-settings=.pipeline/mavenGlobalSettings.xml", params) }) t.Run("ProjectSettingsFile and GlobalSettingsFile https url", func(t *testing.T) { - config := codeqlExecuteScanOptions{BuildTool: "maven", BuildCommand: "mvn clean install", GlobalSettingsFile: "https://jenkins-sap-test.com/test.xml", ProjectSettingsFile: "http://jenkins-sap-test.com/test.xml"} - params := getMavenSettings(&config, newCodeqlExecuteScanTestsUtils()) + config := codeqlExecuteScanOptions{BuildTool: "maven", GlobalSettingsFile: "https://jenkins-sap-test.com/test.xml", ProjectSettingsFile: "http://jenkins-sap-test.com/test.xml"} + buildCmd := "mvn clean install" + params := getMavenSettings(buildCmd, &config, newCodeqlExecuteScanTestsUtils()) assert.Equal(t, " --global-settings=.pipeline/mavenGlobalSettings.xml --settings=.pipeline/mavenProjectSettings.xml", params) }) t.Run("ProjectSettingsFile and GlobalSettingsFile http url", func(t *testing.T) { - config := codeqlExecuteScanOptions{BuildTool: "maven", BuildCommand: "mvn clean install", GlobalSettingsFile: "http://jenkins-sap-test.com/test.xml", ProjectSettingsFile: "http://jenkins-sap-test.com/test.xml"} - params := getMavenSettings(&config, newCodeqlExecuteScanTestsUtils()) + config := codeqlExecuteScanOptions{BuildTool: "maven", GlobalSettingsFile: "http://jenkins-sap-test.com/test.xml", ProjectSettingsFile: "http://jenkins-sap-test.com/test.xml"} + buildCmd := "mvn clean install" + params := getMavenSettings(buildCmd, &config, newCodeqlExecuteScanTestsUtils()) assert.Equal(t, " --global-settings=.pipeline/mavenGlobalSettings.xml --settings=.pipeline/mavenProjectSettings.xml", params) }) t.Run("ProjectSettingsFile file and GlobalSettingsFile https url", func(t *testing.T) { - config := codeqlExecuteScanOptions{BuildTool: "maven", BuildCommand: "mvn clean install", GlobalSettingsFile: "https://jenkins-sap-test.com/test.xml", ProjectSettingsFile: "test.xml"} - params := getMavenSettings(&config, newCodeqlExecuteScanTestsUtils()) + config := codeqlExecuteScanOptions{BuildTool: "maven", GlobalSettingsFile: "https://jenkins-sap-test.com/test.xml", ProjectSettingsFile: "test.xml"} + buildCmd := "mvn clean install" + params := getMavenSettings(buildCmd, &config, newCodeqlExecuteScanTestsUtils()) assert.Equal(t, " --global-settings=.pipeline/mavenGlobalSettings.xml --settings=test.xml", params) }) t.Run("ProjectSettingsFile file and GlobalSettingsFile https url", func(t *testing.T) { - config := codeqlExecuteScanOptions{BuildTool: "maven", BuildCommand: "mvn clean install", GlobalSettingsFile: "http://jenkins-sap-test.com/test.xml", ProjectSettingsFile: "test.xml"} - params := getMavenSettings(&config, newCodeqlExecuteScanTestsUtils()) + config := codeqlExecuteScanOptions{BuildTool: "maven", GlobalSettingsFile: "http://jenkins-sap-test.com/test.xml", ProjectSettingsFile: "test.xml"} + buildCmd := "mvn clean install" + params := getMavenSettings(buildCmd, &config, newCodeqlExecuteScanTestsUtils()) assert.Equal(t, " --global-settings=.pipeline/mavenGlobalSettings.xml --settings=test.xml", params) }) t.Run("ProjectSettingsFile https url and GlobalSettingsFile file", func(t *testing.T) { - config := codeqlExecuteScanOptions{BuildTool: "maven", BuildCommand: "mvn clean install", GlobalSettingsFile: "global.xml", ProjectSettingsFile: "http://jenkins-sap-test.com/test.xml"} - params := getMavenSettings(&config, newCodeqlExecuteScanTestsUtils()) + config := codeqlExecuteScanOptions{BuildTool: "maven", GlobalSettingsFile: "global.xml", ProjectSettingsFile: "http://jenkins-sap-test.com/test.xml"} + buildCmd := "mvn clean install" + params := getMavenSettings(buildCmd, &config, newCodeqlExecuteScanTestsUtils()) assert.Equal(t, " --global-settings=global.xml --settings=.pipeline/mavenProjectSettings.xml", params) }) t.Run("ProjectSettingsFile http url and GlobalSettingsFile file", func(t *testing.T) { - config := codeqlExecuteScanOptions{BuildTool: "maven", BuildCommand: "mvn clean install", GlobalSettingsFile: "global.xml", ProjectSettingsFile: "http://jenkins-sap-test.com/test.xml"} - params := getMavenSettings(&config, newCodeqlExecuteScanTestsUtils()) + config := codeqlExecuteScanOptions{BuildTool: "maven", GlobalSettingsFile: "global.xml", ProjectSettingsFile: "http://jenkins-sap-test.com/test.xml"} + buildCmd := "mvn clean install" + params := getMavenSettings(buildCmd, &config, newCodeqlExecuteScanTestsUtils()) assert.Equal(t, " --global-settings=global.xml --settings=.pipeline/mavenProjectSettings.xml", params) }) } +func TestUpdateCmdFlag(t *testing.T) { + t.Parallel() + + t.Run("No maven", func(t *testing.T) { + config := codeqlExecuteScanOptions{BuildTool: "npm"} + customFlags := map[string]string{ + "--command": "mvn clean install", + } + updateCmdFlag(&config, customFlags, newCodeqlExecuteScanTestsUtils()) + assert.Equal(t, "mvn clean install", customFlags["--command"]) + assert.Equal(t, "", customFlags["-c"]) + }) + + t.Run("No custom flags with build command provided", func(t *testing.T) { + config := codeqlExecuteScanOptions{BuildTool: "maven", ProjectSettingsFile: "test.xml", GlobalSettingsFile: "global.xml"} + customFlags := map[string]string{} + updateCmdFlag(&config, customFlags, newCodeqlExecuteScanTestsUtils()) + assert.Equal(t, "", customFlags["--command"]) + assert.Equal(t, "", customFlags["-c"]) + }) + + t.Run("Both --command and -c flags are set, no settings file provided", func(t *testing.T) { + config := codeqlExecuteScanOptions{BuildTool: "maven"} + customFlags := map[string]string{ + "--command": "mvn clean install", + "-c": "mvn clean install -DskipTests", + } + updateCmdFlag(&config, customFlags, newCodeqlExecuteScanTestsUtils()) + assert.Equal(t, "mvn clean install", customFlags["--command"]) + assert.Equal(t, "", customFlags["-c"]) + }) + + t.Run("Only -c flag is set, no settings file provided", func(t *testing.T) { + config := codeqlExecuteScanOptions{BuildTool: "maven"} + customFlags := map[string]string{ + "-c": "mvn clean install -DskipTests", + } + updateCmdFlag(&config, customFlags, newCodeqlExecuteScanTestsUtils()) + assert.Equal(t, "mvn clean install -DskipTests", customFlags["--command"]) + assert.Equal(t, "", customFlags["-c"]) + }) + + t.Run("Update custom command with GlobalSettingsFile and ProjectSettingsFile", func(t *testing.T) { + config := codeqlExecuteScanOptions{BuildTool: "maven", ProjectSettingsFile: "test.xml", GlobalSettingsFile: "global.xml"} + customFlags := map[string]string{ + "--command": "mvn clean install", + } + updateCmdFlag(&config, customFlags, newCodeqlExecuteScanTestsUtils()) + assert.Equal(t, "mvn clean install --global-settings=global.xml --settings=test.xml", customFlags["--command"]) + assert.Equal(t, "", customFlags["-c"]) + }) + + t.Run("Custom command has --global-settings and --settings", func(t *testing.T) { + config := codeqlExecuteScanOptions{BuildTool: "maven", ProjectSettingsFile: "test.xml", GlobalSettingsFile: "global.xml"} + customFlags := map[string]string{ + "--command": "mvn clean install --settings=test1.xml --global-settings=global1.xml", + } + updateCmdFlag(&config, customFlags, newCodeqlExecuteScanTestsUtils()) + assert.Equal(t, "mvn clean install --settings=test1.xml --global-settings=global1.xml", customFlags["--command"]) + assert.Equal(t, "", customFlags["-c"]) + }) +} + func TestAddDataToInfluxDB(t *testing.T) { repoUrl := "https://github.htllo.test/Testing/codeql" repoRef := "https://github.htllo.test/Testing/codeql/tree/branch" @@ -489,6 +568,268 @@ func TestAddDataToInfluxDB(t *testing.T) { }) } +func TestPrepareCmdForDatabaseCreate(t *testing.T) { + t.Parallel() + + t.Run("No custom flags", func(t *testing.T) { + config := &codeqlExecuteScanOptions{ + Database: "codeqlDB", + ModulePath: "./", + BuildTool: "maven", + BuildCommand: "mvn clean install", + } + cmd, err := prepareCmdForDatabaseCreate(map[string]string{}, config, newCodeqlExecuteScanTestsUtils()) + assert.NoError(t, err) + assert.NotEmpty(t, cmd) + assert.Equal(t, 10, len(cmd)) + assert.Equal(t, "database create codeqlDB --overwrite --source-root . --working-dir ./ --language=java --command=mvn clean install", + strings.Join(cmd, " ")) + }) + + t.Run("No custom flags, custom build tool", func(t *testing.T) { + config := &codeqlExecuteScanOptions{ + Database: "codeqlDB", + ModulePath: "./", + BuildTool: "custom", + Language: "javascript", + } + cmd, err := prepareCmdForDatabaseCreate(map[string]string{}, config, newCodeqlExecuteScanTestsUtils()) + assert.NoError(t, err) + assert.NotEmpty(t, cmd) + assert.Equal(t, 9, len(cmd)) + assert.Equal(t, "database create codeqlDB --overwrite --source-root . --working-dir ./ --language=javascript", + strings.Join(cmd, " ")) + }) + + t.Run("No custom flags, custom build tool, no language specified", func(t *testing.T) { + config := &codeqlExecuteScanOptions{ + Database: "codeqlDB", + ModulePath: "./", + BuildTool: "custom", + } + _, err := prepareCmdForDatabaseCreate(map[string]string{}, config, newCodeqlExecuteScanTestsUtils()) + assert.Error(t, err) + }) + + t.Run("No custom flags, invalid build tool, no language specified", func(t *testing.T) { + config := &codeqlExecuteScanOptions{ + Database: "codeqlDB", + ModulePath: "./", + BuildTool: "test", + } + _, err := prepareCmdForDatabaseCreate(map[string]string{}, config, newCodeqlExecuteScanTestsUtils()) + assert.Error(t, err) + }) + + t.Run("No custom flags, invalid build tool, language specified", func(t *testing.T) { + config := &codeqlExecuteScanOptions{ + Database: "codeqlDB", + ModulePath: "./", + BuildTool: "test", + Language: "javascript", + } + cmd, err := prepareCmdForDatabaseCreate(map[string]string{}, config, newCodeqlExecuteScanTestsUtils()) + assert.NoError(t, err) + assert.NotEmpty(t, cmd) + assert.Equal(t, 9, len(cmd)) + assert.Equal(t, "database create codeqlDB --overwrite --source-root . --working-dir ./ --language=javascript", + strings.Join(cmd, " ")) + }) + + t.Run("Custom flags, overwriting source-root", func(t *testing.T) { + config := &codeqlExecuteScanOptions{ + Database: "codeqlDB", + ModulePath: "./", + Language: "javascript", + } + customFlags := map[string]string{ + "--source-root": "--source-root=customSrcRoot/", + } + cmd, err := prepareCmdForDatabaseCreate(customFlags, config, newCodeqlExecuteScanTestsUtils()) + assert.NoError(t, err) + assert.NotEmpty(t, cmd) + assert.Equal(t, 8, len(cmd)) + assert.Equal(t, "database create codeqlDB --overwrite --working-dir ./ --language=javascript --source-root=customSrcRoot/", + strings.Join(cmd, " ")) + }) + + t.Run("Custom flags, overwriting threads from config", func(t *testing.T) { + config := &codeqlExecuteScanOptions{ + Database: "codeqlDB", + ModulePath: "./", + Language: "javascript", + Threads: "0", + Ram: "2000", + } + customFlags := map[string]string{ + "--source-root": "--source-root=customSrcRoot/", + "-j": "-j=1", + } + cmd, err := prepareCmdForDatabaseCreate(customFlags, config, newCodeqlExecuteScanTestsUtils()) + assert.NoError(t, err) + assert.NotEmpty(t, cmd) + assert.Equal(t, 10, len(cmd)) + assert.True(t, "database create codeqlDB --overwrite --working-dir ./ --language=javascript --ram=2000 -j=1 --source-root=customSrcRoot/" == strings.Join(cmd, " ") || + "database create codeqlDB --overwrite --working-dir ./ --language=javascript --ram=2000 --source-root=customSrcRoot/ -j=1" == strings.Join(cmd, " ")) + }) + +} + +func TestPrepareCmdForDatabaseAnalyze(t *testing.T) { + t.Parallel() + + t.Run("No additional flags, no querySuite, sarif format", func(t *testing.T) { + config := &codeqlExecuteScanOptions{ + Database: "codeqlDB", + } + cmd, err := prepareCmdForDatabaseAnalyze(map[string]string{}, config, "sarif-latest", "codeqlReport.sarif") + assert.NoError(t, err) + assert.NotEmpty(t, cmd) + assert.Equal(t, 5, len(cmd)) + assert.Equal(t, "database analyze --format=sarif-latest --output=target/codeqlReport.sarif codeqlDB", strings.Join(cmd, " ")) + }) + + t.Run("No additional flags, no querySuite, csv format", func(t *testing.T) { + config := &codeqlExecuteScanOptions{ + Database: "codeqlDB", + } + cmd, err := prepareCmdForDatabaseAnalyze(map[string]string{}, config, "csv", "codeqlReport.csv") + assert.NoError(t, err) + assert.NotEmpty(t, cmd) + assert.Equal(t, 5, len(cmd)) + assert.Equal(t, "database analyze --format=csv --output=target/codeqlReport.csv codeqlDB", strings.Join(cmd, " ")) + }) + + t.Run("No additional flags, set querySuite", func(t *testing.T) { + config := &codeqlExecuteScanOptions{ + Database: "codeqlDB", + QuerySuite: "security.ql", + } + cmd, err := prepareCmdForDatabaseAnalyze(map[string]string{}, config, "sarif-latest", "codeqlReport.sarif") + assert.NoError(t, err) + assert.NotEmpty(t, cmd) + assert.Equal(t, 6, len(cmd)) + assert.Equal(t, "database analyze --format=sarif-latest --output=target/codeqlReport.sarif codeqlDB security.ql", strings.Join(cmd, " ")) + }) + + t.Run("No custom flags, flags from config", func(t *testing.T) { + config := &codeqlExecuteScanOptions{ + Database: "codeqlDB", + QuerySuite: "security.ql", + Threads: "1", + Ram: "2000", + } + cmd, err := prepareCmdForDatabaseAnalyze(map[string]string{}, config, "sarif-latest", "codeqlReport.sarif") + assert.NoError(t, err) + assert.NotEmpty(t, cmd) + assert.Equal(t, 8, len(cmd)) + assert.Equal(t, "database analyze --format=sarif-latest --output=target/codeqlReport.sarif codeqlDB --threads=1 --ram=2000 security.ql", strings.Join(cmd, " ")) + }) + + t.Run("Custom flags, overwriting threads", func(t *testing.T) { + config := &codeqlExecuteScanOptions{ + Database: "codeqlDB", + QuerySuite: "security.ql", + Threads: "1", + Ram: "2000", + } + customFlags := map[string]string{ + "--threads": "--threads=2", + } + cmd, err := prepareCmdForDatabaseAnalyze(customFlags, config, "sarif-latest", "codeqlReport.sarif") + assert.NoError(t, err) + assert.NotEmpty(t, cmd) + assert.Equal(t, 8, len(cmd)) + assert.Equal(t, "database analyze --format=sarif-latest --output=target/codeqlReport.sarif codeqlDB --ram=2000 --threads=2 security.ql", strings.Join(cmd, " ")) + }) + + t.Run("Custom flags, overwriting threads (-j)", func(t *testing.T) { + config := &codeqlExecuteScanOptions{ + Database: "codeqlDB", + QuerySuite: "security.ql", + Threads: "1", + Ram: "2000", + } + customFlags := map[string]string{ + "-j": "-j=2", + } + cmd, err := prepareCmdForDatabaseAnalyze(customFlags, config, "sarif-latest", "codeqlReport.sarif") + assert.NoError(t, err) + assert.NotEmpty(t, cmd) + assert.Equal(t, 8, len(cmd)) + assert.Equal(t, "database analyze --format=sarif-latest --output=target/codeqlReport.sarif codeqlDB --ram=2000 -j=2 security.ql", strings.Join(cmd, " ")) + }) + + t.Run("Custom flags, no overwriting", func(t *testing.T) { + config := &codeqlExecuteScanOptions{ + Database: "codeqlDB", + QuerySuite: "security.ql", + Threads: "1", + Ram: "2000", + } + customFlags := map[string]string{ + "--no-download": "--no-download", + } + cmd, err := prepareCmdForDatabaseAnalyze(customFlags, config, "sarif-latest", "codeqlReport.sarif") + assert.NoError(t, err) + assert.NotEmpty(t, cmd) + assert.Equal(t, 9, len(cmd)) + assert.Equal(t, "database analyze --format=sarif-latest --output=target/codeqlReport.sarif codeqlDB --threads=1 --ram=2000 --no-download security.ql", strings.Join(cmd, " ")) + }) +} + +func TestPrepareCmdForUploadResults(t *testing.T) { + t.Parallel() + + config := &codeqlExecuteScanOptions{ + ModulePath: "./", + } + + t.Run("All configs are set", func(t *testing.T) { + repoInfo := &codeql.RepoInfo{ + CommitId: "commitId", + ServerUrl: "http://github.com", + Repo: "repo", + Owner: "owner", + Ref: "refs/heads/branch", + } + cmd := prepareCmdForUploadResults(config, repoInfo, "token") + assert.NotEmpty(t, cmd) + assert.Equal(t, 8, len(cmd)) + }) + + t.Run("Configs are set partially", func(t *testing.T) { + repoInfo := &codeql.RepoInfo{ + CommitId: "commitId", + ServerUrl: "http://github.com", + Repo: "repo", + } + cmd := prepareCmdForUploadResults(config, repoInfo, "token") + assert.NotEmpty(t, cmd) + assert.Equal(t, 6, len(cmd)) + }) + + t.Run("Empty token", func(t *testing.T) { + repoInfo := &codeql.RepoInfo{ + CommitId: "commitId", + ServerUrl: "http://github.com", + Repo: "repo", + Owner: "owner", + Ref: "refs/heads/branch", + } + cmd := prepareCmdForUploadResults(config, repoInfo, "") + assert.NotEmpty(t, cmd) + assert.Equal(t, 7, len(cmd)) + }) + + t.Run("Empty configs and token", func(t *testing.T) { + repoInfo := &codeql.RepoInfo{} + cmd := prepareCmdForUploadResults(config, repoInfo, "") + assert.NotEmpty(t, cmd) + assert.Equal(t, 3, len(cmd)) + }) +} + type CodeqlSarifUploaderMock struct { counter int } diff --git a/pkg/codeql/flags.go b/pkg/codeql/flags.go new file mode 100644 index 0000000000..636980c35e --- /dev/null +++ b/pkg/codeql/flags.go @@ -0,0 +1,128 @@ +package codeql + +import ( + "strings" + + "github.com/SAP/jenkins-library/pkg/log" +) + +var longShortFlagsMap = map[string]string{ + "--language": "-l", + "--command": "-c", + "--source-root": "-s", + "--github-url": "-g", + "--mode": "-m", + "--extractor-option": "-O", + "--github-auth-stdin": "-a", + "--threads": "-j", + "--ram": "-M", +} + +func IsFlagSetByUser(customFlags map[string]string, flagsToCheck []string) bool { + for _, flag := range flagsToCheck { + if _, exists := customFlags[flag]; exists { + return true + } + } + return false +} + +func AppendFlagIfNotSetByUser(cmd []string, flagToCheck []string, flagToAppend []string, customFlags map[string]string) []string { + if !IsFlagSetByUser(customFlags, flagToCheck) { + cmd = append(cmd, flagToAppend...) + } + return cmd +} + +func AppendCustomFlags(cmd []string, flags map[string]string) []string { + for _, flag := range flags { + if strings.TrimSpace(flag) != "" { + cmd = append(cmd, flag) + } + } + return cmd +} + +// parseFlags parses the input string and extracts individual flags. +// Flags can have values, which can be enclosed in single quotes (”) or double quotes (""). +// Flags are separated by whitespace. +// The function returns a slice of strings, where each string represents a flag or a flag with its value. +func parseFlags(input string) []string { + result := []string{} + isFlagStarted := false + isString := false + flag := "" + for i, c := range input { + if !isFlagStarted { + if string(c) == " " { + continue + } + flag += string(c) + isFlagStarted = true + continue + } + if string(c) == "\"" || string(c) == "'" { + if i == len(input)-1 { + continue + } + if !isString { + isString = true + + } else { + result = append(result, flag) + flag = "" + isFlagStarted = false + isString = false + } + continue + } + if string(c) == " " && !isString { + result = append(result, flag) + flag = "" + isFlagStarted = false + continue + } + flag += string(c) + } + result = append(result, flag) + return result +} + +func removeDuplicateFlags(customFlags map[string]string, shortFlags map[string]string) { + for longFlag, shortFlag := range shortFlags { + if _, longExists := customFlags[longFlag]; longExists { + if _, shortExists := customFlags[shortFlag]; shortExists { + log.Entry().Warnf("Both forms of the flag %s and %s are presented, %s will be ignored.", + longFlag, shortFlag, customFlags[shortFlag]) + delete(customFlags, shortFlag) + } + } + } +} + +func ParseCustomFlags(flagsStr string) map[string]string { + flagsMap := make(map[string]string) + parsedFlags := parseFlags(flagsStr) + + for _, flag := range parsedFlags { + if strings.Contains(flag, "=") { + split := strings.SplitN(flag, "=", 2) + flagsMap[split[0]] = flag + } else { + flagsMap[flag] = flag + } + } + + removeDuplicateFlags(flagsMap, longShortFlagsMap) + return flagsMap +} + +func AppendThreadsAndRam(cmd []string, threads, ram string, customFlags map[string]string) []string { + if len(threads) > 0 && !IsFlagSetByUser(customFlags, []string{"--threads", "-j"}) { + cmd = append(cmd, "--threads="+threads) + } + if len(ram) > 0 && !IsFlagSetByUser(customFlags, []string{"--ram", "-M"}) { + cmd = append(cmd, "--ram="+ram) + } + return cmd +} diff --git a/pkg/codeql/flags_test.go b/pkg/codeql/flags_test.go new file mode 100644 index 0000000000..aa5834c4de --- /dev/null +++ b/pkg/codeql/flags_test.go @@ -0,0 +1,329 @@ +package codeql + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestIsFlagSetByUser(t *testing.T) { + t.Parallel() + + customFlags := map[string]string{ + "--flag1": "--flag1=1", + "-f2": "-f2=2", + "--flag3": "--flag3", + } + + t.Run("Flag is not set by user", func(t *testing.T) { + input := []string{"-f4"} + assert.False(t, IsFlagSetByUser(customFlags, input)) + }) + t.Run("Flag is set by user", func(t *testing.T) { + input := []string{"-f2"} + assert.True(t, IsFlagSetByUser(customFlags, input)) + }) + t.Run("One of flags is set by user", func(t *testing.T) { + input := []string{"--flag2", "-f2"} + assert.True(t, IsFlagSetByUser(customFlags, input)) + }) +} + +func TestAppendFlagIfNotSetByUser(t *testing.T) { + t.Parallel() + + t.Run("Flag is not set by user", func(t *testing.T) { + result := []string{} + flagsToCheck := []string{"--flag1", "-f1"} + flagToAppend := []string{"--flag1=1"} + customFlags := map[string]string{ + "--flag2": "--flag2=1", + } + result = AppendFlagIfNotSetByUser(result, flagsToCheck, flagToAppend, customFlags) + assert.Equal(t, 1, len(result)) + assert.Equal(t, "--flag1=1", result[0]) + }) + + t.Run("Flag is set by user", func(t *testing.T) { + result := []string{} + flagsToCheck := []string{"--flag1", "-f1"} + flagToAppend := []string{"--flag1=1"} + customFlags := map[string]string{ + "--flag1": "--flag1=2", + } + result = AppendFlagIfNotSetByUser(result, flagsToCheck, flagToAppend, customFlags) + assert.Equal(t, 0, len(result)) + }) +} + +func TestAppendCustomFlags(t *testing.T) { + t.Parallel() + + t.Run("Flags with values", func(t *testing.T) { + flags := map[string]string{ + "--flag1": "--flag1=1", + "--flag2": "--flag2=2", + "--flag3": "--flag3=3", + } + result := []string{} + result = AppendCustomFlags(result, flags) + assert.Equal(t, 3, len(result)) + jointFlags := strings.Join(result, " ") + assert.True(t, strings.Contains(jointFlags, "--flag1=1")) + assert.True(t, strings.Contains(jointFlags, "--flag2=2")) + assert.True(t, strings.Contains(jointFlags, "--flag3=3")) + }) + t.Run("Flags without values", func(t *testing.T) { + flags := map[string]string{ + "--flag1": "--flag1", + "--flag2": "--flag2", + "--flag3": "--flag3", + } + result := []string{} + result = AppendCustomFlags(result, flags) + assert.Equal(t, 3, len(result)) + jointFlags := strings.Join(result, " ") + assert.True(t, strings.Contains(jointFlags, "--flag1")) + assert.True(t, strings.Contains(jointFlags, "--flag2")) + assert.True(t, strings.Contains(jointFlags, "--flag3")) + }) + t.Run("Some flags without values", func(t *testing.T) { + flags := map[string]string{ + "--flag1": "--flag1=1", + "--flag2": "--flag2=1", + "--flag3": "--flag3", + } + result := []string{} + result = AppendCustomFlags(result, flags) + assert.Equal(t, 3, len(result)) + jointFlags := strings.Join(result, " ") + assert.True(t, strings.Contains(jointFlags, "--flag1=1")) + assert.True(t, strings.Contains(jointFlags, "--flag2=1")) + assert.True(t, strings.Contains(jointFlags, "--flag3")) + }) + t.Run("Empty input", func(t *testing.T) { + flags := map[string]string{} + expected := []string{} + result := []string{} + result = AppendCustomFlags(result, flags) + assert.Equal(t, expected, result) + }) +} + +func TestParseFlags(t *testing.T) { + t.Parallel() + + t.Run("Valid flags with values", func(t *testing.T) { + inputStr := "--flag1=1 --flag2=2 --flag3=string" + expected := map[string]bool{ + "--flag1=1": true, + "--flag2=2": true, + "--flag3=string": true, + } + result := parseFlags(inputStr) + assert.Equal(t, len(expected), len(result)) + for _, f := range result { + assert.True(t, expected[f]) + } + }) + + t.Run("Valid flags without values", func(t *testing.T) { + inputStr := "--flag1 -flag2 -f3" + expected := map[string]bool{ + "--flag1": true, + "-flag2": true, + "-f3": true, + } + result := parseFlags(inputStr) + assert.Equal(t, len(expected), len(result)) + for _, f := range result { + assert.True(t, expected[f]) + } + }) + + t.Run("Valid flags with spaces in value", func(t *testing.T) { + inputStr := "--flag1='mvn install' --flag2=\"mvn clean install\" -f3='mvn clean install -DskipTests=true'" + expected := map[string]bool{ + "--flag1=mvn install": true, + "--flag2=mvn clean install": true, + "-f3=mvn clean install -DskipTests=true": true, + } + result := parseFlags(inputStr) + assert.Equal(t, len(expected), len(result)) + for _, f := range result { + assert.True(t, expected[f]) + } + }) +} + +func TestRemoveDuplicateFlags(t *testing.T) { + t.Parallel() + + longShortFlags := map[string]string{ + "--flag1": "-f1", + "--flag2": "-f2", + "--flag3": "-f3", + } + + t.Run("No duplications", func(t *testing.T) { + flags := map[string]string{ + "--flag1": "--flag1=1", + "-f2": "-f2=2", + "--flag3": "--flag3", + } + expected := map[string]string{ + "--flag1": "--flag1=1", + "-f2": "-f2=2", + "--flag3": "--flag3", + } + removeDuplicateFlags(flags, longShortFlags) + assert.Equal(t, len(expected), len(flags)) + for k, v := range flags { + assert.Equal(t, expected[k], v) + } + }) + + t.Run("Duplications", func(t *testing.T) { + flags := map[string]string{ + "--flag1": "--flag1=1", + "-f1": "-f1=2", + "--flag2": "--flag2=1", + "-f2": "-f2=2", + "--flag3": "--flag3", + "-f3": "-f3", + } + expected := map[string]string{ + "--flag1": "--flag1=1", + "--flag2": "--flag2=1", + "--flag3": "--flag3", + } + removeDuplicateFlags(flags, longShortFlags) + assert.Equal(t, len(expected), len(flags)) + for k, v := range flags { + assert.Equal(t, expected[k], v) + } + }) +} + +func TestParseCustomFlags(t *testing.T) { + t.Parallel() + + t.Run("Valid flags with values", func(t *testing.T) { + inputStr := "--flag1=1 --flag2=2 --flag3=string" + expected := map[string]bool{ + "--flag1=1": true, + "--flag2=2": true, + "--flag3=string": true, + } + result := ParseCustomFlags(inputStr) + assert.Equal(t, len(expected), len(result)) + for _, f := range result { + assert.True(t, expected[f]) + } + }) + + t.Run("Valid flags with duplication", func(t *testing.T) { + inputStr := "--flag1=1 --flag2=2 --flag3=string --flag2=3" + expected := map[string]bool{ + "--flag1=1": true, + "--flag2=3": true, + "--flag3=string": true, + } + result := ParseCustomFlags(inputStr) + assert.Equal(t, len(expected), len(result)) + for _, f := range result { + assert.True(t, expected[f]) + } + }) + + t.Run("Valid flags with duplicated short flag", func(t *testing.T) { + inputStr := "--flag1=1 --flag2=2 --flag3=string --language=java -l=python" + expected := map[string]bool{ + "--flag1=1": true, + "--flag2=2": true, + "--flag3=string": true, + "--language=java": true, + } + result := ParseCustomFlags(inputStr) + assert.Equal(t, len(expected), len(result)) + for _, f := range result { + assert.True(t, expected[f]) + } + }) + + t.Run("Valid flags without values", func(t *testing.T) { + inputStr := "--flag1 -flag2 -f3" + expected := map[string]bool{ + "--flag1": true, + "-flag2": true, + "-f3": true, + } + result := ParseCustomFlags(inputStr) + assert.Equal(t, len(expected), len(result)) + for _, f := range result { + assert.True(t, expected[f]) + } + }) + + t.Run("Valid flags with spaces in value", func(t *testing.T) { + inputStr := "--flag1='mvn install' --flag2=\"mvn clean install\" -f3='mvn clean install -DskipTests=true'" + expected := map[string]bool{ + "--flag1=mvn install": true, + "--flag2=mvn clean install": true, + "-f3=mvn clean install -DskipTests=true": true, + } + result := ParseCustomFlags(inputStr) + assert.Equal(t, len(expected), len(result)) + for _, f := range result { + assert.True(t, expected[f]) + } + }) +} + +func TestAppendThreadsAndRam(t *testing.T) { + t.Parallel() + + threads := "0" + ram := "2000" + + t.Run("Threads and ram are set by user", func(t *testing.T) { + customFlags := map[string]string{ + "--threads": "--threads=1", + "--ram": "--ram=3000", + } + params := []string{} + params = AppendThreadsAndRam(params, threads, ram, customFlags) + assert.Equal(t, 0, len(params)) + }) + + t.Run("Threads and ram are not set by user", func(t *testing.T) { + customFlags := map[string]string{} + params := []string{} + params = AppendThreadsAndRam(params, threads, ram, customFlags) + assert.Equal(t, 2, len(params)) + paramsStr := strings.Join(params, " ") + assert.True(t, strings.Contains(paramsStr, "--threads=0")) + assert.True(t, strings.Contains(paramsStr, "--ram=2000")) + }) + + t.Run("Threads is set by user, ram is not", func(t *testing.T) { + customFlags := map[string]string{ + "--threads": "--threads=1", + } + params := []string{} + params = AppendThreadsAndRam(params, threads, ram, customFlags) + assert.Equal(t, 1, len(params)) + assert.True(t, strings.Contains(params[0], "--ram=2000")) + }) + + t.Run("Add params to non-empty slice", func(t *testing.T) { + customFlags := map[string]string{} + params := []string{"cmd"} + params = AppendThreadsAndRam(params, threads, ram, customFlags) + assert.Equal(t, 3, len(params)) + assert.Equal(t, "cmd", params[0]) + assert.Equal(t, "--threads=0", params[1]) + assert.Equal(t, "--ram=2000", params[2]) + }) +} diff --git a/resources/metadata/codeqlExecuteScan.yaml b/resources/metadata/codeqlExecuteScan.yaml index f0d91acc5c..f489299d71 100644 --- a/resources/metadata/codeqlExecuteScan.yaml +++ b/resources/metadata/codeqlExecuteScan.yaml @@ -209,6 +209,30 @@ spec: - PARAMETERS aliases: - name: maven/globalSettingsFile + - name: databaseCreateFlags + type: string + description: "A space-separated string of flags for the 'codeql database create' command." + longDescription: |- + A space-separated string of flags for the 'codeql database create' command. + + If both long and short forms of the same flag are provided, the long form takes precedence. + Example input: "--threads=1 --ram=2000" + scope: + - STEPS + - STAGES + - PARAMETERS + - name: databaseAnalyzeFlags + type: string + description: "A space-separated string of flags for the 'codeql database analyze' command." + longDescription: |- + A space-separated string of flags for the 'codeql database analyze' command. + + If both long and short forms of the same flag are provided, the long form takes precedence. + Example input: "--threads=1 --ram=2000" + scope: + - STEPS + - STAGES + - PARAMETERS containers: - image: "" outputs: