diff --git a/.ci/check_new_rules.go b/.ci/check_new_rules.go index 54793243..ada58f4d 100644 --- a/.ci/check_new_rules.go +++ b/.ci/check_new_rules.go @@ -11,7 +11,7 @@ import ( ) var ( - regexGitleaksRules = regexp.MustCompile(`configRules\s*=\s*append\(configRules,\s*rules\.([a-zA-Z0-9_]+)\(`) + regexGitleaksRules = regexp.MustCompile(`^[^/\n\r]configRules\s*=\s*append\(configRules,\s*rules\.([a-zA-Z0-9_]+)\(`) regex2msRules = regexp.MustCompile(`allRules\s*=\s*append\(allRules,\s*Rule{Rule:\s*\*rules\.([a-zA-Z0-9_]+)\(\),`) ) @@ -61,7 +61,7 @@ func main() { os.Exit(1) } else { - fmt.Printf("No differences found.") + fmt.Println("No differences found.") os.Exit(0) } } diff --git a/go.mod b/go.mod index 96131a61..4bee2ae0 100644 --- a/go.mod +++ b/go.mod @@ -4,14 +4,14 @@ go 1.20 require ( github.com/bwmarrin/discordgo v0.27.1 - github.com/gitleaks/go-gitdiff v0.8.0 + github.com/gitleaks/go-gitdiff v0.9.0 github.com/rs/zerolog v1.29.0 github.com/slack-go/slack v0.12.2 github.com/spf13/cobra v1.6.1 github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.15.0 github.com/stretchr/testify v1.8.1 - github.com/zricethezav/gitleaks/v8 v8.17.1-0.20230717122715-f0dcd4d9cfe9 + github.com/zricethezav/gitleaks/v8 v8.18.0 golang.org/x/time v0.1.0 gopkg.in/yaml.v2 v2.4.0 ) diff --git a/go.sum b/go.sum index 11b55700..0e6e0940 100644 --- a/go.sum +++ b/go.sum @@ -68,8 +68,8 @@ github.com/fatih/semgroup v1.2.0/go.mod h1:1KAD4iIYfXjE4U13B48VM4z9QUwV5Tt8O4rS8 github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= -github.com/gitleaks/go-gitdiff v0.8.0 h1:7aExTZm+K/M/EQKOyYcub8rIAdWK6ONxPGuRzxmWW+0= -github.com/gitleaks/go-gitdiff v0.8.0/go.mod h1:pKz0X4YzCKZs30BL+weqBIG7mx0jl4tF1uXV9ZyNvrA= +github.com/gitleaks/go-gitdiff v0.9.0 h1:SHAU2l0ZBEo8g82EeFewhVy81sb7JCxW76oSPtR/Nqg= +github.com/gitleaks/go-gitdiff v0.9.0/go.mod h1:pKz0X4YzCKZs30BL+weqBIG7mx0jl4tF1uXV9ZyNvrA= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -227,8 +227,8 @@ github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/zricethezav/gitleaks/v8 v8.17.1-0.20230717122715-f0dcd4d9cfe9 h1:gw0iPgtVuWBW1XQoZed9Y0rWaZ9la1qOooa6aRHsEFo= -github.com/zricethezav/gitleaks/v8 v8.17.1-0.20230717122715-f0dcd4d9cfe9/go.mod h1:/0z7cslO7d0y29YRvHgYefeTu7UIqOmx95A4wMhcQtE= +github.com/zricethezav/gitleaks/v8 v8.18.0 h1:+zXcDpHATT9E/eA9UZqcKNW/O1mg882NLmO/6z4CFK0= +github.com/zricethezav/gitleaks/v8 v8.18.0/go.mod h1:JulwKdEMpiOxVFQxZFFixY51QzDZPn1xJ1/p7YqX4hQ= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= diff --git a/lib/channels.go b/lib/channels.go new file mode 100644 index 00000000..ac01cfeb --- /dev/null +++ b/lib/channels.go @@ -0,0 +1,12 @@ +package lib + +import "sync" + +func BindChannels[T any](source <-chan T, dest chan<- T, wg *sync.WaitGroup) { + if wg != nil { + defer wg.Done() + } + for item := range source { + dest <- item + } +} diff --git a/plugins/git.go b/plugins/git.go index a8f20d2a..2761dcca 100644 --- a/plugins/git.go +++ b/plugins/git.go @@ -6,6 +6,7 @@ import ( "strings" "sync" + "github.com/checkmarx/2ms/lib" "github.com/gitleaks/go-gitdiff/gitdiff" "github.com/rs/zerolog/log" "github.com/spf13/cobra" @@ -68,13 +69,10 @@ func (p *GitPlugin) buildScanOptions() string { } func (p *GitPlugin) scanGit(path string, scanOptions string, itemsChan chan Item, errChan chan error) { - fileChan, err := git.GitLog(path, scanOptions) - if err != nil { - errChan <- fmt.Errorf("error while scanning git repository: %w", err) - } - log.Debug().Msgf("scanned git repository: %s", path) + diffs, close := p.readGitLog(path, scanOptions, errChan) + defer close() - for file := range fileChan { + for file := range diffs { log.Debug().Msgf("file: %s; Commit: %s", file.NewName, file.PatchHeader.Title) if file.IsBinary || file.IsDelete { continue @@ -97,6 +95,25 @@ func (p *GitPlugin) scanGit(path string, scanOptions string, itemsChan chan Item } } +func (p *GitPlugin) readGitLog(path string, scanOptions string, errChan chan error) (<-chan *gitdiff.File, func()) { + gitLog, err := git.NewGitLogCmd(path, scanOptions) + if err != nil { + errChan <- fmt.Errorf("error while scanning git repository: %w", err) + } + wait := func() { + err := gitLog.Wait() + if err != nil { + errChan <- fmt.Errorf("error while waiting for git log to finish: %w", err) + } + } + log.Debug().Msgf("scanning git repository: %s", path) + + p.WaitGroup.Add(1) + go lib.BindChannels[error](gitLog.ErrCh(), errChan, p.WaitGroup) + + return gitLog.DiffFilesCh(), wait +} + func validGitRepoArgs(cmd *cobra.Command, args []string) error { stat, err := os.Stat(args[0]) if err != nil { diff --git a/secrets/rules/rules.go b/secrets/rules/rules.go index 1e7a03e6..cb55617c 100644 --- a/secrets/rules/rules.go +++ b/secrets/rules/rules.go @@ -41,6 +41,7 @@ func getDefaultRules() *[]Rule { allRules = append(allRules, Rule{Rule: *rules.AsanaClientID(), Tags: []string{TagClientId}}) allRules = append(allRules, Rule{Rule: *rules.AsanaClientSecret(), Tags: []string{TagClientSecret}}) allRules = append(allRules, Rule{Rule: *rules.Atlassian(), Tags: []string{TagApiToken}}) + allRules = append(allRules, Rule{Rule: *rules.Authress(), Tags: []string{TagAccessToken}}) allRules = append(allRules, Rule{Rule: *rules.AWS(), Tags: []string{TagAccessToken}}) allRules = append(allRules, Rule{Rule: *rules.BitBucketClientID(), Tags: []string{TagClientId}}) allRules = append(allRules, Rule{Rule: *rules.BitBucketClientSecret(), Tags: []string{TagClientSecret}}) @@ -55,6 +56,7 @@ func getDefaultRules() *[]Rule { allRules = append(allRules, Rule{Rule: *rules.Contentful(), Tags: []string{TagApiToken}}) allRules = append(allRules, Rule{Rule: *rules.Databricks(), Tags: []string{TagApiToken}}) allRules = append(allRules, Rule{Rule: *rules.DatadogtokenAccessToken(), Tags: []string{TagAccessToken, TagClientId}}) + allRules = append(allRules, Rule{Rule: *rules.DefinedNetworkingAPIToken(), Tags: []string{TagApiToken}}) allRules = append(allRules, Rule{Rule: *rules.DigitalOceanPAT(), Tags: []string{TagAccessToken}}) allRules = append(allRules, Rule{Rule: *rules.DigitalOceanOAuthToken(), Tags: []string{TagAccessToken}}) allRules = append(allRules, Rule{Rule: *rules.DigitalOceanRefreshToken(), Tags: []string{TagRefreshToken}}) @@ -129,6 +131,7 @@ func getDefaultRules() *[]Rule { allRules = append(allRules, Rule{Rule: *rules.NPM(), Tags: []string{TagAccessToken}}) allRules = append(allRules, Rule{Rule: *rules.NytimesAccessToken(), Tags: []string{TagAccessToken}}) allRules = append(allRules, Rule{Rule: *rules.OktaAccessToken(), Tags: []string{TagAccessToken}}) + allRules = append(allRules, Rule{Rule: *rules.OpenAI(), Tags: []string{TagApiKey}}) allRules = append(allRules, Rule{Rule: *rules.PlaidAccessID(), Tags: []string{TagClientId}}) allRules = append(allRules, Rule{Rule: *rules.PlaidSecretKey(), Tags: []string{TagSecretKey}}) allRules = append(allRules, Rule{Rule: *rules.PlaidAccessToken(), Tags: []string{TagApiToken}}) @@ -169,6 +172,7 @@ func getDefaultRules() *[]Rule { allRules = append(allRules, Rule{Rule: *rules.SquareSpaceAccessToken(), Tags: []string{TagAccessToken}}) allRules = append(allRules, Rule{Rule: *rules.SumoLogicAccessID(), Tags: []string{TagAccessId}}) allRules = append(allRules, Rule{Rule: *rules.SumoLogicAccessToken(), Tags: []string{TagAccessToken}}) + allRules = append(allRules, Rule{Rule: *rules.Snyk(), Tags: []string{TagApiKey}}) allRules = append(allRules, Rule{Rule: *rules.TeamsWebhook(), Tags: []string{TagWebhook}}) allRules = append(allRules, Rule{Rule: *rules.TelegramBotToken(), Tags: []string{TagApiToken}}) allRules = append(allRules, Rule{Rule: *rules.TravisCIAccessToken(), Tags: []string{TagAccessToken}})