From 45e330973af57711a54200d6843a066156cc1f47 Mon Sep 17 00:00:00 2001 From: "Baruch Odem (Rothkoff)" Date: Thu, 28 Sep 2023 12:14:17 +0300 Subject: [PATCH] feat: add max-target-megabytes to ignore large files (#187) ``` --max-target-megabytes int files larger than this will be skipped. Omit or set to 0 to disable this check. ``` --- README.md | 2 ++ cmd/main.go | 34 ++++++++++++++--------------- plugins/filesystem.go | 1 + secrets/secrets.go | 23 +++++++++++++------- secrets/secrets_test.go | 48 ++++++++++++++++++++++------------------- 5 files changed, 61 insertions(+), 47 deletions(-) diff --git a/README.md b/README.md index 59ce2ee7..53dd1480 100644 --- a/README.md +++ b/README.md @@ -127,6 +127,8 @@ Flags: --ignore-result strings ignore specific result by id --ignore-rule strings ignore rules by name or tag --log-level string log level (trace, debug, info, warn, error, fatal) (default "info") + --max-target-megabytes int files larger than this will be skipped. + Omit or set to 0 to disable this check. --regex stringArray custom regexes to apply to the scan, must be valid Go regex --report-path strings path to generate report files. The output format will be determined by the file extension (.json, .yaml, .sarif) --rule strings select rules by name or tag to apply to this scan diff --git a/cmd/main.go b/cmd/main.go index eb77f3a7..e9e0624a 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -20,15 +20,16 @@ const ( outputFormatRegexpPattern = `^(ya?ml|json|sarif)$` configFileFlag = "config" - logLevelFlagName = "log-level" - reportPathFlagName = "report-path" - stdoutFormatFlagName = "stdout-format" - customRegexRuleFlagName = "regex" - ruleFlagName = "rule" - ignoreRuleFlagName = "ignore-rule" - ignoreFlagName = "ignore-result" - specialRulesFlagName = "add-special-rule" - ignoreOnExitFlagName = "ignore-on-exit" + logLevelFlagName = "log-level" + reportPathFlagName = "report-path" + stdoutFormatFlagName = "stdout-format" + customRegexRuleFlagName = "regex" + ruleFlagName = "rule" + ignoreRuleFlagName = "ignore-rule" + ignoreFlagName = "ignore-result" + specialRulesFlagName = "add-special-rule" + ignoreOnExitFlagName = "ignore-on-exit" + maxTargetMegabytesFlagName = "max-target-megabytes" ) var ( @@ -36,11 +37,9 @@ var ( reportPathVar []string stdoutFormatVar string customRegexRuleVar []string - ruleVar []string - ignoreRuleVar []string ignoreVar []string - specialRulesVar []string ignoreOnExitVar = ignoreOnExitNone + secretsConfigVar secrets.SecretsConfig ) var rootCmd = &cobra.Command{ @@ -84,13 +83,14 @@ func Execute() (int, error) { rootCmd.PersistentFlags().StringSliceVar(&reportPathVar, reportPathFlagName, []string{}, "path to generate report files. The output format will be determined by the file extension (.json, .yaml, .sarif)") rootCmd.PersistentFlags().StringVar(&stdoutFormatVar, stdoutFormatFlagName, "yaml", "stdout output format, available formats are: json, yaml, sarif") rootCmd.PersistentFlags().StringArrayVar(&customRegexRuleVar, customRegexRuleFlagName, []string{}, "custom regexes to apply to the scan, must be valid Go regex") - rootCmd.PersistentFlags().StringSliceVar(&ruleVar, ruleFlagName, []string{}, "select rules by name or tag to apply to this scan") - rootCmd.PersistentFlags().StringSliceVar(&ignoreRuleVar, ignoreRuleFlagName, []string{}, "ignore rules by name or tag") + rootCmd.PersistentFlags().StringSliceVar(&secretsConfigVar.SelectedList, ruleFlagName, []string{}, "select rules by name or tag to apply to this scan") + rootCmd.PersistentFlags().StringSliceVar(&secretsConfigVar.IgnoreList, ignoreRuleFlagName, []string{}, "ignore rules by name or tag") rootCmd.PersistentFlags().StringSliceVar(&ignoreVar, ignoreFlagName, []string{}, "ignore specific result by id") - rootCmd.PersistentFlags().StringSliceVar(&specialRulesVar, specialRulesFlagName, []string{}, "special (non-default) rules to apply.\nThis list is not affected by the --rule and --ignore-rule flags.") + rootCmd.PersistentFlags().StringSliceVar(&secretsConfigVar.SpecialList, specialRulesFlagName, []string{}, "special (non-default) rules to apply.\nThis list is not affected by the --rule and --ignore-rule flags.") rootCmd.PersistentFlags().Var(&ignoreOnExitVar, ignoreOnExitFlagName, "defines which kind of non-zero exits code should be ignored\naccepts: all, results, errors, none\nexample: if 'results' is set, only engine errors will make 2ms exit code different from 0") + rootCmd.PersistentFlags().IntVar(&secretsConfigVar.MaxTargetMegabytes, maxTargetMegabytesFlagName, 0, "files larger than this will be skipped.\nOmit or set to 0 to disable this check.") - rootCmd.AddCommand(secrets.GetRulesCommand(&ruleVar, &ignoreRuleVar, &specialRulesVar)) + rootCmd.AddCommand(secrets.GetRulesCommand(&secretsConfigVar)) group := "Commands" rootCmd.AddGroup(&cobra.Group{Title: group, ID: group}) @@ -120,7 +120,7 @@ func preRun(cmd *cobra.Command, args []string) error { return err } - secrets, err := secrets.Init(ruleVar, ignoreRuleVar, specialRulesVar) + secrets, err := secrets.Init(secretsConfigVar) if err != nil { return err } diff --git a/plugins/filesystem.go b/plugins/filesystem.go index a17b803b..6556ec13 100644 --- a/plugins/filesystem.go +++ b/plugins/filesystem.go @@ -115,6 +115,7 @@ func (p *FileSystemPlugin) getItems(items chan Item, errs chan error, wg *sync.W } func (p *FileSystemPlugin) getItem(wg *sync.WaitGroup, filePath string) (*Item, error) { + log.Debug().Str("file", filePath).Msg("reading file") b, err := os.ReadFile(filePath) if err != nil { return nil, err diff --git a/secrets/secrets.go b/secrets/secrets.go index 0742ea20..5809eb3c 100644 --- a/secrets/secrets.go +++ b/secrets/secrets.go @@ -26,8 +26,16 @@ type Secrets struct { const customRegexRuleIdFormat = "custom-regex-%d" -func Init(selectedList, ignoreList, specialList []string) (*Secrets, error) { - selectedRules := rules.FilterRules(selectedList, ignoreList, specialList) +type SecretsConfig struct { + SelectedList []string + IgnoreList []string + SpecialList []string + + MaxTargetMegabytes int +} + +func Init(secretsConfig SecretsConfig) (*Secrets, error) { + selectedRules := rules.FilterRules(secretsConfig.SelectedList, secretsConfig.IgnoreList, secretsConfig.SpecialList) if len(*selectedRules) == 0 { return nil, fmt.Errorf("no rules were selected") } @@ -39,11 +47,10 @@ func Init(selectedList, ignoreList, specialList []string) (*Secrets, error) { rulesToBeApplied[rule.Rule.RuleID] = rule.Rule } - config := config.Config{ + detector := detect.NewDetector(config.Config{ Rules: rulesToBeApplied, - } - - detector := detect.NewDetector(config) + }) + detector.MaxTargetMegaBytes = secretsConfig.MaxTargetMegabytes return &Secrets{ rules: rulesToBeApplied, @@ -109,14 +116,14 @@ func isSecretIgnored(secret *reporting.Secret, ignoredIds *[]string) bool { return false } -func GetRulesCommand(selectedList, ignoreList, specialList *[]string) *cobra.Command { +func GetRulesCommand(secretsConfig *SecretsConfig) *cobra.Command { return &cobra.Command{ Use: "rules", Short: "List all rules", Long: `List all rules`, RunE: func(cmd *cobra.Command, args []string) error { - rules := rules.FilterRules(*selectedList, *ignoreList, *specialList) + rules := rules.FilterRules(secretsConfig.SelectedList, secretsConfig.IgnoreList, secretsConfig.SpecialList) tab := tabwriter.NewWriter(os.Stdout, 1, 2, 2, ' ', 0) diff --git a/secrets/secrets_test.go b/secrets/secrets_test.go index d94e2042..44bdccb6 100644 --- a/secrets/secrets_test.go +++ b/secrets/secrets_test.go @@ -15,38 +15,42 @@ func Test_Init(t *testing.T) { specialRule := rules.HardcodedPassword() tests := []struct { - name string - selectedList []string - ignoreList []string - specialList []string - expectedErr error + name string + secretsConfig SecretsConfig + expectedErr error }{ { - name: "selected and ignore flags used together for the same rule", - selectedList: []string{allRules[0].Rule.RuleID}, - ignoreList: []string{allRules[0].Rule.RuleID}, - specialList: []string{}, - expectedErr: fmt.Errorf("no rules were selected"), + name: "selected and ignore flags used together for the same rule", + secretsConfig: SecretsConfig{ + SelectedList: []string{allRules[0].Rule.RuleID}, + IgnoreList: []string{allRules[0].Rule.RuleID}, + SpecialList: []string{}, + }, + expectedErr: fmt.Errorf("no rules were selected"), }, { - name: "non existent select flag", - selectedList: []string{"non-existent-tag-name"}, - ignoreList: []string{}, - specialList: []string{"non-existent-tag-name"}, - expectedErr: fmt.Errorf("no rules were selected"), + name: "non existent select flag", + secretsConfig: SecretsConfig{ + SelectedList: []string{"non-existent-tag-name"}, + IgnoreList: []string{}, + SpecialList: []string{"non-existent-tag-name"}, + }, + expectedErr: fmt.Errorf("no rules were selected"), }, { - name: "exiting special rule", - selectedList: []string{"non-existent-tag-name"}, - ignoreList: []string{}, - specialList: []string{specialRule.RuleID}, - expectedErr: nil, + name: "exiting special rule", + secretsConfig: SecretsConfig{ + SelectedList: []string{"non-existent-tag-name"}, + IgnoreList: []string{}, + SpecialList: []string{specialRule.RuleID}, + }, + expectedErr: nil, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - _, err := Init(test.selectedList, test.ignoreList, test.specialList) + _, err := Init(test.secretsConfig) if err == nil && test.expectedErr != nil { t.Errorf("expected error, got nil") } @@ -107,7 +111,7 @@ func TestSecrets(t *testing.T) { }, } - detector, err := Init([]string{}, []string{}, []string{}) + detector, err := Init(SecretsConfig{}) if err != nil { t.Fatal(err) }