feat: add max-target-megabytes to ignore large files (#187)

```
--max-target-megabytes int   files larger than this will be skipped.
                             Omit or set to 0 to disable this check.
```

README.md

@@ -127,6 +127,8 @@ Flags:
       --ignore-result strings         ignore specific result by id
       --ignore-rule strings           ignore rules by name or tag
       --log-level string              log level (trace, debug, info, warn, error, fatal) (default "info")
+      --max-target-megabytes int      files larger than this will be skipped.
+                                      Omit or set to 0 to disable this check.
       --regex stringArray             custom regexes to apply to the scan, must be valid Go regex
       --report-path strings           path to generate report files. The output format will be determined by the file extension (.json, .yaml, .sarif)
       --rule strings                  select rules by name or tag to apply to this scan

cmd/main.go

@@ -20,27 +20,26 @@ const (
 	outputFormatRegexpPattern = `^(ya?ml|json|sarif)$`
 	configFileFlag            = "config"
 
-	logLevelFlagName        = "log-level"
-	reportPathFlagName      = "report-path"
-	stdoutFormatFlagName    = "stdout-format"
-	customRegexRuleFlagName = "regex"
-	ruleFlagName            = "rule"
-	ignoreRuleFlagName      = "ignore-rule"
-	ignoreFlagName          = "ignore-result"
-	specialRulesFlagName    = "add-special-rule"
-	ignoreOnExitFlagName    = "ignore-on-exit"
+	logLevelFlagName           = "log-level"
+	reportPathFlagName         = "report-path"
+	stdoutFormatFlagName       = "stdout-format"
+	customRegexRuleFlagName    = "regex"
+	ruleFlagName               = "rule"
+	ignoreRuleFlagName         = "ignore-rule"
+	ignoreFlagName             = "ignore-result"
+	specialRulesFlagName       = "add-special-rule"
+	ignoreOnExitFlagName       = "ignore-on-exit"
+	maxTargetMegabytesFlagName = "max-target-megabytes"
 )
 
 var (
 	logLevelVar        string
 	reportPathVar      []string
 	stdoutFormatVar    string
 	customRegexRuleVar []string
-	ruleVar            []string
-	ignoreRuleVar      []string
 	ignoreVar          []string
-	specialRulesVar    []string
 	ignoreOnExitVar    = ignoreOnExitNone
+	secretsConfigVar   secrets.SecretsConfig
 )
 
 var rootCmd = &cobra.Command{
@@ -84,13 +83,14 @@ func Execute() (int, error) {
 	rootCmd.PersistentFlags().StringSliceVar(&reportPathVar, reportPathFlagName, []string{}, "path to generate report files. The output format will be determined by the file extension (.json, .yaml, .sarif)")
 	rootCmd.PersistentFlags().StringVar(&stdoutFormatVar, stdoutFormatFlagName, "yaml", "stdout output format, available formats are: json, yaml, sarif")
 	rootCmd.PersistentFlags().StringArrayVar(&customRegexRuleVar, customRegexRuleFlagName, []string{}, "custom regexes to apply to the scan, must be valid Go regex")
-	rootCmd.PersistentFlags().StringSliceVar(&ruleVar, ruleFlagName, []string{}, "select rules by name or tag to apply to this scan")
-	rootCmd.PersistentFlags().StringSliceVar(&ignoreRuleVar, ignoreRuleFlagName, []string{}, "ignore rules by name or tag")
+	rootCmd.PersistentFlags().StringSliceVar(&secretsConfigVar.SelectedList, ruleFlagName, []string{}, "select rules by name or tag to apply to this scan")
+	rootCmd.PersistentFlags().StringSliceVar(&secretsConfigVar.IgnoreList, ignoreRuleFlagName, []string{}, "ignore rules by name or tag")
 	rootCmd.PersistentFlags().StringSliceVar(&ignoreVar, ignoreFlagName, []string{}, "ignore specific result by id")
-	rootCmd.PersistentFlags().StringSliceVar(&specialRulesVar, specialRulesFlagName, []string{}, "special (non-default) rules to apply.\nThis list is not affected by the --rule and --ignore-rule flags.")
+	rootCmd.PersistentFlags().StringSliceVar(&secretsConfigVar.SpecialList, specialRulesFlagName, []string{}, "special (non-default) rules to apply.\nThis list is not affected by the --rule and --ignore-rule flags.")
 	rootCmd.PersistentFlags().Var(&ignoreOnExitVar, ignoreOnExitFlagName, "defines which kind of non-zero exits code should be ignored\naccepts: all, results, errors, none\nexample: if 'results' is set, only engine errors will make 2ms exit code different from 0")
+	rootCmd.PersistentFlags().IntVar(&secretsConfigVar.MaxTargetMegabytes, maxTargetMegabytesFlagName, 0, "files larger than this will be skipped.\nOmit or set to 0 to disable this check.")
 
-	rootCmd.AddCommand(secrets.GetRulesCommand(&ruleVar, &ignoreRuleVar, &specialRulesVar))
+	rootCmd.AddCommand(secrets.GetRulesCommand(&secretsConfigVar))
 
 	group := "Commands"
 	rootCmd.AddGroup(&cobra.Group{Title: group, ID: group})
@@ -120,7 +120,7 @@ func preRun(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
-	secrets, err := secrets.Init(ruleVar, ignoreRuleVar, specialRulesVar)
+	secrets, err := secrets.Init(secretsConfigVar)
 	if err != nil {
 		return err
 	}

plugins/filesystem.go

@@ -115,6 +115,7 @@ func (p *FileSystemPlugin) getItems(items chan Item, errs chan error, wg *sync.W
 }
 
 func (p *FileSystemPlugin) getItem(wg *sync.WaitGroup, filePath string) (*Item, error) {
+	log.Debug().Str("file", filePath).Msg("reading file")
 	b, err := os.ReadFile(filePath)
 	if err != nil {
 		return nil, err

secrets/secrets.go

@@ -26,8 +26,16 @@ type Secrets struct {
 
 const customRegexRuleIdFormat = "custom-regex-%d"
 
-func Init(selectedList, ignoreList, specialList []string) (*Secrets, error) {
-	selectedRules := rules.FilterRules(selectedList, ignoreList, specialList)
+type SecretsConfig struct {
+	SelectedList []string
+	IgnoreList   []string
+	SpecialList  []string
+
+	MaxTargetMegabytes int
+}
+
+func Init(secretsConfig SecretsConfig) (*Secrets, error) {
+	selectedRules := rules.FilterRules(secretsConfig.SelectedList, secretsConfig.IgnoreList, secretsConfig.SpecialList)
 	if len(*selectedRules) == 0 {
 		return nil, fmt.Errorf("no rules were selected")
 	}
@@ -39,11 +47,10 @@ func Init(selectedList, ignoreList, specialList []string) (*Secrets, error) {
 		rulesToBeApplied[rule.Rule.RuleID] = rule.Rule
 	}
 
-	config := config.Config{
+	detector := detect.NewDetector(config.Config{
 		Rules: rulesToBeApplied,
-	}
-
-	detector := detect.NewDetector(config)
+	})
+	detector.MaxTargetMegaBytes = secretsConfig.MaxTargetMegabytes
 
 	return &Secrets{
 		rules:    rulesToBeApplied,
@@ -109,14 +116,14 @@ func isSecretIgnored(secret *reporting.Secret, ignoredIds *[]string) bool {
 	return false
 }
 
-func GetRulesCommand(selectedList, ignoreList, specialList *[]string) *cobra.Command {
+func GetRulesCommand(secretsConfig *SecretsConfig) *cobra.Command {
 	return &cobra.Command{
 		Use:   "rules",
 		Short: "List all rules",
 		Long:  `List all rules`,
 		RunE: func(cmd *cobra.Command, args []string) error {
 
-			rules := rules.FilterRules(*selectedList, *ignoreList, *specialList)
+			rules := rules.FilterRules(secretsConfig.SelectedList, secretsConfig.IgnoreList, secretsConfig.SpecialList)
 
 			tab := tabwriter.NewWriter(os.Stdout, 1, 2, 2, ' ', 0)
 

secrets/secrets_test.go

@@ -15,38 +15,42 @@ func Test_Init(t *testing.T) {
 	specialRule := rules.HardcodedPassword()
 
 	tests := []struct {
-		name         string
-		selectedList []string
-		ignoreList   []string
-		specialList  []string
-		expectedErr  error
+		name          string
+		secretsConfig SecretsConfig
+		expectedErr   error
 	}{
 		{
-			name:         "selected and ignore flags used together for the same rule",
-			selectedList: []string{allRules[0].Rule.RuleID},
-			ignoreList:   []string{allRules[0].Rule.RuleID},
-			specialList:  []string{},
-			expectedErr:  fmt.Errorf("no rules were selected"),
+			name: "selected and ignore flags used together for the same rule",
+			secretsConfig: SecretsConfig{
+				SelectedList: []string{allRules[0].Rule.RuleID},
+				IgnoreList:   []string{allRules[0].Rule.RuleID},
+				SpecialList:  []string{},
+			},
+			expectedErr: fmt.Errorf("no rules were selected"),
 		},
 		{
-			name:         "non existent select flag",
-			selectedList: []string{"non-existent-tag-name"},
-			ignoreList:   []string{},
-			specialList:  []string{"non-existent-tag-name"},
-			expectedErr:  fmt.Errorf("no rules were selected"),
+			name: "non existent select flag",
+			secretsConfig: SecretsConfig{
+				SelectedList: []string{"non-existent-tag-name"},
+				IgnoreList:   []string{},
+				SpecialList:  []string{"non-existent-tag-name"},
+			},
+			expectedErr: fmt.Errorf("no rules were selected"),
 		},
 		{
-			name:         "exiting special rule",
-			selectedList: []string{"non-existent-tag-name"},
-			ignoreList:   []string{},
-			specialList:  []string{specialRule.RuleID},
-			expectedErr:  nil,
+			name: "exiting special rule",
+			secretsConfig: SecretsConfig{
+				SelectedList: []string{"non-existent-tag-name"},
+				IgnoreList:   []string{},
+				SpecialList:  []string{specialRule.RuleID},
+			},
+			expectedErr: nil,
 		},
 	}
 
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			_, err := Init(test.selectedList, test.ignoreList, test.specialList)
+			_, err := Init(test.secretsConfig)
 			if err == nil && test.expectedErr != nil {
 				t.Errorf("expected error, got nil")
 			}
@@ -107,7 +111,7 @@ func TestSecrets(t *testing.T) {
 		},
 	}
 
-	detector, err := Init([]string{}, []string{}, []string{})
+	detector, err := Init(SecretsConfig{})
 	if err != nil {
 		t.Fatal(err)
 	}