From 5ebd3f2ad39f9d417ddab0bfb8564139b2c93eb8 Mon Sep 17 00:00:00 2001 From: binyamin2 Date: Tue, 12 Sep 2023 12:38:13 +0300 Subject: [PATCH 1/7] Update version to "gitleaks" dependency --- go.mod | 4 ++-- go.sum | 6 ++++-- secrets/secrets.go | 5 +++++ 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 96131a61..4bee2ae0 100644 --- a/go.mod +++ b/go.mod @@ -4,14 +4,14 @@ go 1.20 require ( github.com/bwmarrin/discordgo v0.27.1 - github.com/gitleaks/go-gitdiff v0.8.0 + github.com/gitleaks/go-gitdiff v0.9.0 github.com/rs/zerolog v1.29.0 github.com/slack-go/slack v0.12.2 github.com/spf13/cobra v1.6.1 github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.15.0 github.com/stretchr/testify v1.8.1 - github.com/zricethezav/gitleaks/v8 v8.17.1-0.20230717122715-f0dcd4d9cfe9 + github.com/zricethezav/gitleaks/v8 v8.18.0 golang.org/x/time v0.1.0 gopkg.in/yaml.v2 v2.4.0 ) diff --git a/go.sum b/go.sum index 11b55700..39c7c096 100644 --- a/go.sum +++ b/go.sum @@ -70,6 +70,8 @@ github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4 github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/gitleaks/go-gitdiff v0.8.0 h1:7aExTZm+K/M/EQKOyYcub8rIAdWK6ONxPGuRzxmWW+0= github.com/gitleaks/go-gitdiff v0.8.0/go.mod h1:pKz0X4YzCKZs30BL+weqBIG7mx0jl4tF1uXV9ZyNvrA= +github.com/gitleaks/go-gitdiff v0.9.0 h1:SHAU2l0ZBEo8g82EeFewhVy81sb7JCxW76oSPtR/Nqg= +github.com/gitleaks/go-gitdiff v0.9.0/go.mod h1:pKz0X4YzCKZs30BL+weqBIG7mx0jl4tF1uXV9ZyNvrA= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -227,8 +229,8 @@ github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/zricethezav/gitleaks/v8 v8.17.1-0.20230717122715-f0dcd4d9cfe9 h1:gw0iPgtVuWBW1XQoZed9Y0rWaZ9la1qOooa6aRHsEFo= -github.com/zricethezav/gitleaks/v8 v8.17.1-0.20230717122715-f0dcd4d9cfe9/go.mod h1:/0z7cslO7d0y29YRvHgYefeTu7UIqOmx95A4wMhcQtE= +github.com/zricethezav/gitleaks/v8 v8.18.0 h1:+zXcDpHATT9E/eA9UZqcKNW/O1mg882NLmO/6z4CFK0= +github.com/zricethezav/gitleaks/v8 v8.18.0/go.mod h1:JulwKdEMpiOxVFQxZFFixY51QzDZPn1xJ1/p7YqX4hQ= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= diff --git a/secrets/secrets.go b/secrets/secrets.go index 56b9f771..5ae64e8c 100644 --- a/secrets/secrets.go +++ b/secrets/secrets.go @@ -196,6 +196,7 @@ func loadAllRules() []Rule { allRules = append(allRules, Rule{Rule: *rules.AsanaClientID(), Tags: []string{TagClientId}}) allRules = append(allRules, Rule{Rule: *rules.AsanaClientSecret(), Tags: []string{TagClientSecret}}) allRules = append(allRules, Rule{Rule: *rules.Atlassian(), Tags: []string{TagApiToken}}) + allRules = append(allRules, Rule{Rule: *rules.Authress(), Tags: []string{TagAccessToken}}) allRules = append(allRules, Rule{Rule: *rules.AWS(), Tags: []string{TagAccessToken}}) allRules = append(allRules, Rule{Rule: *rules.BitBucketClientID(), Tags: []string{TagClientId}}) allRules = append(allRules, Rule{Rule: *rules.BitBucketClientSecret(), Tags: []string{TagClientSecret}}) @@ -210,6 +211,7 @@ func loadAllRules() []Rule { allRules = append(allRules, Rule{Rule: *rules.Contentful(), Tags: []string{TagApiToken}}) allRules = append(allRules, Rule{Rule: *rules.Databricks(), Tags: []string{TagApiToken}}) allRules = append(allRules, Rule{Rule: *rules.DatadogtokenAccessToken(), Tags: []string{TagAccessToken, TagClientId}}) + allRules = append(allRules, Rule{Rule: *rules.DefinedNetworkingAPIToken(), Tags: []string{TagApiToken}}) allRules = append(allRules, Rule{Rule: *rules.DigitalOceanPAT(), Tags: []string{TagAccessToken}}) allRules = append(allRules, Rule{Rule: *rules.DigitalOceanOAuthToken(), Tags: []string{TagAccessToken}}) allRules = append(allRules, Rule{Rule: *rules.DigitalOceanRefreshToken(), Tags: []string{TagRefreshToken}}) @@ -238,6 +240,7 @@ func loadAllRules() []Rule { allRules = append(allRules, Rule{Rule: *rules.FrameIO(), Tags: []string{TagApiToken}}) allRules = append(allRules, Rule{Rule: *rules.FreshbooksAccessToken(), Tags: []string{TagAccessToken}}) allRules = append(allRules, Rule{Rule: *rules.GCPAPIKey(), Tags: []string{TagApiKey}}) + allRules = append(allRules, Rule{Rule: *rules.GCPServiceAccount(), Tags: []string{TagAccessToken}}) allRules = append(allRules, Rule{Rule: *rules.GenericCredential(), Tags: []string{TagApiKey}}) allRules = append(allRules, Rule{Rule: *rules.GitHubPat(), Tags: []string{TagAccessToken}}) allRules = append(allRules, Rule{Rule: *rules.GitHubFineGrainedPat(), Tags: []string{TagAccessToken}}) @@ -284,6 +287,7 @@ func loadAllRules() []Rule { allRules = append(allRules, Rule{Rule: *rules.NPM(), Tags: []string{TagAccessToken}}) allRules = append(allRules, Rule{Rule: *rules.NytimesAccessToken(), Tags: []string{TagAccessToken}}) allRules = append(allRules, Rule{Rule: *rules.OktaAccessToken(), Tags: []string{TagAccessToken}}) + allRules = append(allRules, Rule{Rule: *rules.OpenAI(), Tags: []string{TagApiKey}}) allRules = append(allRules, Rule{Rule: *rules.PlaidAccessID(), Tags: []string{TagClientId}}) allRules = append(allRules, Rule{Rule: *rules.PlaidSecretKey(), Tags: []string{TagSecretKey}}) allRules = append(allRules, Rule{Rule: *rules.PlaidAccessToken(), Tags: []string{TagApiToken}}) @@ -324,6 +328,7 @@ func loadAllRules() []Rule { allRules = append(allRules, Rule{Rule: *rules.SquareSpaceAccessToken(), Tags: []string{TagAccessToken}}) allRules = append(allRules, Rule{Rule: *rules.SumoLogicAccessID(), Tags: []string{TagAccessId}}) allRules = append(allRules, Rule{Rule: *rules.SumoLogicAccessToken(), Tags: []string{TagAccessToken}}) + allRules = append(allRules, Rule{Rule: *rules.Snyk(), Tags: []string{TagApiKey}}) allRules = append(allRules, Rule{Rule: *rules.TeamsWebhook(), Tags: []string{TagWebhook}}) allRules = append(allRules, Rule{Rule: *rules.TelegramBotToken(), Tags: []string{TagApiToken}}) allRules = append(allRules, Rule{Rule: *rules.TravisCIAccessToken(), Tags: []string{TagAccessToken}}) From 1c7ff0d6e2b6ae2597ec29555aeb88c23152f09d Mon Sep 17 00:00:00 2001 From: binyamin2 Date: Tue, 12 Sep 2023 12:52:00 +0300 Subject: [PATCH 2/7] fix - irrelevant change in the last commit --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 4bee2ae0..0c52bcd9 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.20 require ( github.com/bwmarrin/discordgo v0.27.1 - github.com/gitleaks/go-gitdiff v0.9.0 + github.com/gitleaks/go-gitdiff v0.8.0 github.com/rs/zerolog v1.29.0 github.com/slack-go/slack v0.12.2 github.com/spf13/cobra v1.6.1 From ac3dd77c0fba5db49e122f87dedc9448aa8038d1 Mon Sep 17 00:00:00 2001 From: binyamin2 Date: Tue, 12 Sep 2023 12:54:50 +0300 Subject: [PATCH 3/7] final fix --- go.sum | 2 -- 1 file changed, 2 deletions(-) diff --git a/go.sum b/go.sum index 39c7c096..380c8c93 100644 --- a/go.sum +++ b/go.sum @@ -70,8 +70,6 @@ github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4 github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/gitleaks/go-gitdiff v0.8.0 h1:7aExTZm+K/M/EQKOyYcub8rIAdWK6ONxPGuRzxmWW+0= github.com/gitleaks/go-gitdiff v0.8.0/go.mod h1:pKz0X4YzCKZs30BL+weqBIG7mx0jl4tF1uXV9ZyNvrA= -github.com/gitleaks/go-gitdiff v0.9.0 h1:SHAU2l0ZBEo8g82EeFewhVy81sb7JCxW76oSPtR/Nqg= -github.com/gitleaks/go-gitdiff v0.9.0/go.mod h1:pKz0X4YzCKZs30BL+weqBIG7mx0jl4tF1uXV9ZyNvrA= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= From cfed47c26ac6f0b4246154d24aebdc4889992adf Mon Sep 17 00:00:00 2001 From: binyamin2 Date: Tue, 12 Sep 2023 13:04:19 +0300 Subject: [PATCH 4/7] final --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 0c52bcd9..4bee2ae0 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.20 require ( github.com/bwmarrin/discordgo v0.27.1 - github.com/gitleaks/go-gitdiff v0.8.0 + github.com/gitleaks/go-gitdiff v0.9.0 github.com/rs/zerolog v1.29.0 github.com/slack-go/slack v0.12.2 github.com/spf13/cobra v1.6.1 diff --git a/go.sum b/go.sum index 380c8c93..0e6e0940 100644 --- a/go.sum +++ b/go.sum @@ -68,8 +68,8 @@ github.com/fatih/semgroup v1.2.0/go.mod h1:1KAD4iIYfXjE4U13B48VM4z9QUwV5Tt8O4rS8 github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= -github.com/gitleaks/go-gitdiff v0.8.0 h1:7aExTZm+K/M/EQKOyYcub8rIAdWK6ONxPGuRzxmWW+0= -github.com/gitleaks/go-gitdiff v0.8.0/go.mod h1:pKz0X4YzCKZs30BL+weqBIG7mx0jl4tF1uXV9ZyNvrA= +github.com/gitleaks/go-gitdiff v0.9.0 h1:SHAU2l0ZBEo8g82EeFewhVy81sb7JCxW76oSPtR/Nqg= +github.com/gitleaks/go-gitdiff v0.9.0/go.mod h1:pKz0X4YzCKZs30BL+weqBIG7mx0jl4tF1uXV9ZyNvrA= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= From 8683e7afccd53dfde36b82606f34c55d505f8417 Mon Sep 17 00:00:00 2001 From: Baruch Odem Date: Thu, 28 Sep 2023 20:35:17 +0300 Subject: [PATCH 5/7] ignore commented out rules in gitleaks --- .ci/check_new_rules.go | 4 ++-- secrets/secrets.go | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.ci/check_new_rules.go b/.ci/check_new_rules.go index f2f0dcab..75b6dabe 100644 --- a/.ci/check_new_rules.go +++ b/.ci/check_new_rules.go @@ -11,7 +11,7 @@ import ( ) var ( - regexGitleaksRules = regexp.MustCompile(`configRules\s*=\s*append\(configRules,\s*rules\.([a-zA-Z0-9_]+)\(`) + regexGitleaksRules = regexp.MustCompile(`^[^/\n\r]configRules\s*=\s*append\(configRules,\s*rules\.([a-zA-Z0-9_]+)\(`) regex2msRules = regexp.MustCompile(`allRules\s*=\s*append\(allRules,\s*Rule{Rule:\s*\*rules\.([a-zA-Z0-9_]+)\(\),`) ) @@ -61,7 +61,7 @@ func main() { os.Exit(1) } else { - fmt.Printf("No differences found.") + fmt.Println("No differences found.") os.Exit(0) } } diff --git a/secrets/secrets.go b/secrets/secrets.go index 5ae64e8c..33799207 100644 --- a/secrets/secrets.go +++ b/secrets/secrets.go @@ -240,7 +240,6 @@ func loadAllRules() []Rule { allRules = append(allRules, Rule{Rule: *rules.FrameIO(), Tags: []string{TagApiToken}}) allRules = append(allRules, Rule{Rule: *rules.FreshbooksAccessToken(), Tags: []string{TagAccessToken}}) allRules = append(allRules, Rule{Rule: *rules.GCPAPIKey(), Tags: []string{TagApiKey}}) - allRules = append(allRules, Rule{Rule: *rules.GCPServiceAccount(), Tags: []string{TagAccessToken}}) allRules = append(allRules, Rule{Rule: *rules.GenericCredential(), Tags: []string{TagApiKey}}) allRules = append(allRules, Rule{Rule: *rules.GitHubPat(), Tags: []string{TagAccessToken}}) allRules = append(allRules, Rule{Rule: *rules.GitHubFineGrainedPat(), Tags: []string{TagAccessToken}}) From 684874168f294822768e5c6ac4bd3fca0524c26a Mon Sep 17 00:00:00 2001 From: Baruch Odem Date: Thu, 28 Sep 2023 20:47:32 +0300 Subject: [PATCH 6/7] use NewGitLogCmd --- lib/channels.go | 12 ++++++++++++ plugins/git.go | 11 ++++++++--- 2 files changed, 20 insertions(+), 3 deletions(-) create mode 100644 lib/channels.go diff --git a/lib/channels.go b/lib/channels.go new file mode 100644 index 00000000..ac01cfeb --- /dev/null +++ b/lib/channels.go @@ -0,0 +1,12 @@ +package lib + +import "sync" + +func BindChannels[T any](source <-chan T, dest chan<- T, wg *sync.WaitGroup) { + if wg != nil { + defer wg.Done() + } + for item := range source { + dest <- item + } +} diff --git a/plugins/git.go b/plugins/git.go index a8f20d2a..99e18924 100644 --- a/plugins/git.go +++ b/plugins/git.go @@ -6,6 +6,7 @@ import ( "strings" "sync" + "github.com/checkmarx/2ms/lib" "github.com/gitleaks/go-gitdiff/gitdiff" "github.com/rs/zerolog/log" "github.com/spf13/cobra" @@ -68,13 +69,17 @@ func (p *GitPlugin) buildScanOptions() string { } func (p *GitPlugin) scanGit(path string, scanOptions string, itemsChan chan Item, errChan chan error) { - fileChan, err := git.GitLog(path, scanOptions) + gitLog, err := git.NewGitLogCmd(path, scanOptions) if err != nil { errChan <- fmt.Errorf("error while scanning git repository: %w", err) } - log.Debug().Msgf("scanned git repository: %s", path) + defer gitLog.Wait() + log.Debug().Msgf("scanning git repository: %s", path) - for file := range fileChan { + p.WaitGroup.Add(1) + go lib.BindChannels[error](gitLog.ErrCh(), errChan, p.WaitGroup) + + for file := range gitLog.DiffFilesCh() { log.Debug().Msgf("file: %s; Commit: %s", file.NewName, file.PatchHeader.Title) if file.IsBinary || file.IsDelete { continue From 3043ef376e631a94f9faf30216ea4dad266f927c Mon Sep 17 00:00:00 2001 From: Baruch Odem Date: Fri, 29 Sep 2023 10:02:13 +0300 Subject: [PATCH 7/7] refactor: gitLog in dedicated function --- plugins/git.go | 32 ++++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/plugins/git.go b/plugins/git.go index 99e18924..2761dcca 100644 --- a/plugins/git.go +++ b/plugins/git.go @@ -69,17 +69,10 @@ func (p *GitPlugin) buildScanOptions() string { } func (p *GitPlugin) scanGit(path string, scanOptions string, itemsChan chan Item, errChan chan error) { - gitLog, err := git.NewGitLogCmd(path, scanOptions) - if err != nil { - errChan <- fmt.Errorf("error while scanning git repository: %w", err) - } - defer gitLog.Wait() - log.Debug().Msgf("scanning git repository: %s", path) + diffs, close := p.readGitLog(path, scanOptions, errChan) + defer close() - p.WaitGroup.Add(1) - go lib.BindChannels[error](gitLog.ErrCh(), errChan, p.WaitGroup) - - for file := range gitLog.DiffFilesCh() { + for file := range diffs { log.Debug().Msgf("file: %s; Commit: %s", file.NewName, file.PatchHeader.Title) if file.IsBinary || file.IsDelete { continue @@ -102,6 +95,25 @@ func (p *GitPlugin) scanGit(path string, scanOptions string, itemsChan chan Item } } +func (p *GitPlugin) readGitLog(path string, scanOptions string, errChan chan error) (<-chan *gitdiff.File, func()) { + gitLog, err := git.NewGitLogCmd(path, scanOptions) + if err != nil { + errChan <- fmt.Errorf("error while scanning git repository: %w", err) + } + wait := func() { + err := gitLog.Wait() + if err != nil { + errChan <- fmt.Errorf("error while waiting for git log to finish: %w", err) + } + } + log.Debug().Msgf("scanning git repository: %s", path) + + p.WaitGroup.Add(1) + go lib.BindChannels[error](gitLog.ErrCh(), errChan, p.WaitGroup) + + return gitLog.DiffFilesCh(), wait +} + func validGitRepoArgs(cmd *cobra.Command, args []string) error { stat, err := os.Stat(args[0]) if err != nil {