From 9c98e20b724b0ff5d87a19ce250854fd6e6fcdf5 Mon Sep 17 00:00:00 2001 From: Baruch Odem Date: Tue, 13 Feb 2024 16:37:41 +0200 Subject: [PATCH 01/18] Add validate flag to root command --- cmd/main.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cmd/main.go b/cmd/main.go index e9e0624a..802d324e 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -30,6 +30,7 @@ const ( specialRulesFlagName = "add-special-rule" ignoreOnExitFlagName = "ignore-on-exit" maxTargetMegabytesFlagName = "max-target-megabytes" + validate = "validate" ) var ( @@ -40,6 +41,7 @@ var ( ignoreVar []string ignoreOnExitVar = ignoreOnExitNone secretsConfigVar secrets.SecretsConfig + validateVar bool ) var rootCmd = &cobra.Command{ @@ -89,6 +91,7 @@ func Execute() (int, error) { rootCmd.PersistentFlags().StringSliceVar(&secretsConfigVar.SpecialList, specialRulesFlagName, []string{}, "special (non-default) rules to apply.\nThis list is not affected by the --rule and --ignore-rule flags.") rootCmd.PersistentFlags().Var(&ignoreOnExitVar, ignoreOnExitFlagName, "defines which kind of non-zero exits code should be ignored\naccepts: all, results, errors, none\nexample: if 'results' is set, only engine errors will make 2ms exit code different from 0") rootCmd.PersistentFlags().IntVar(&secretsConfigVar.MaxTargetMegabytes, maxTargetMegabytesFlagName, 0, "files larger than this will be skipped.\nOmit or set to 0 to disable this check.") + rootCmd.PersistentFlags().BoolVar(&validateVar, validate, false, "Validate the secrets found") rootCmd.AddCommand(secrets.GetRulesCommand(&secretsConfigVar)) From 753c515f6643a3b3b5f56a900384b5499eeeda6f Mon Sep 17 00:00:00 2001 From: Baruch Odem Date: Tue, 13 Feb 2024 16:49:48 +0200 Subject: [PATCH 02/18] Refactor reporting package to use pointers for Secret struct --- cmd/main.go | 2 +- reporting/report.go | 8 ++++---- reporting/report_test.go | 6 +++--- reporting/sarif.go | 2 +- secrets/secrets.go | 6 +++--- secrets/secrets_test.go | 2 +- 6 files changed, 13 insertions(+), 13 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index 802d324e..9edacb44 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -72,7 +72,7 @@ var channels = plugins.Channels{ } var report = reporting.Init() -var secretsChan = make(chan reporting.Secret) +var secretsChan = make(chan *reporting.Secret) func Execute() (int, error) { vConfig.SetEnvPrefix(envPrefix) diff --git a/reporting/report.go b/reporting/report.go index e2a75ff7..1be8437e 100644 --- a/reporting/report.go +++ b/reporting/report.go @@ -17,9 +17,9 @@ const ( ) type Report struct { - TotalItemsScanned int `json:"totalItemsScanned"` - TotalSecretsFound int `json:"totalSecretsFound"` - Results map[string][]Secret `json:"results"` + TotalItemsScanned int `json:"totalItemsScanned"` + TotalSecretsFound int `json:"totalSecretsFound"` + Results map[string][]*Secret `json:"results"` } type Secret struct { @@ -35,7 +35,7 @@ type Secret struct { func Init() *Report { return &Report{ - Results: make(map[string][]Secret), + Results: make(map[string][]*Secret), } } diff --git a/reporting/report_test.go b/reporting/report_test.go index 44a08423..27576b4d 100644 --- a/reporting/report_test.go +++ b/reporting/report_test.go @@ -23,9 +23,9 @@ JPcHeO7M6FohKgcEHX84koQDN98J/L7pFlSoU7WOl6f8BKavIdeSTPS9qQYWdQuT 4Xgur9w/aLZrLM3DSatR+kL+cVTyDTtgCt9Dc8k48Q== -----END RSA PRIVATE KEY-----`) - results := map[string][]Secret{} + results := map[string][]*Secret{} report := Report{len(results), 1, results} - secret := Secret{Source: "bla", StartLine: 0, StartColumn: 0, EndLine: 0, EndColumn: 0, Value: secretValue} + secret := &Secret{Source: "bla", StartLine: 0, StartColumn: 0, EndLine: 0, EndColumn: 0, Value: secretValue} source := "directory\\rawStringAsFile.txt" report.Results[source] = append(report.Results[source], secret) @@ -36,6 +36,6 @@ JPcHeO7M6FohKgcEHX84koQDN98J/L7pFlSoU7WOl6f8BKavIdeSTPS9qQYWdQuT } if !reflect.DeepEqual(report.Results, results) { - t.Errorf("got %q want %q", key, results) + t.Errorf("got %+v want %+v", key, results) } } diff --git a/reporting/sarif.go b/reporting/sarif.go index a26c3d60..db2f13f3 100644 --- a/reporting/sarif.go +++ b/reporting/sarif.go @@ -74,7 +74,7 @@ func getResults(report Report) []Results { return results } -func getLocation(secret Secret) []Locations { +func getLocation(secret *Secret) []Locations { return []Locations{ { PhysicalLocation: PhysicalLocation{ diff --git a/secrets/secrets.go b/secrets/secrets.go index 5809eb3c..2b6e71e2 100644 --- a/secrets/secrets.go +++ b/secrets/secrets.go @@ -58,7 +58,7 @@ func Init(secretsConfig SecretsConfig) (*Secrets, error) { }, nil } -func (s *Secrets) Detect(item plugins.Item, secretsChannel chan reporting.Secret, wg *sync.WaitGroup, ignoredIds []string) { +func (s *Secrets) Detect(item plugins.Item, secretsChannel chan *reporting.Secret, wg *sync.WaitGroup, ignoredIds []string) { defer wg.Done() fragment := detect.Fragment{ @@ -66,7 +66,7 @@ func (s *Secrets) Detect(item plugins.Item, secretsChannel chan reporting.Secret } for _, value := range s.detector.Detect(fragment) { itemId := getFindingId(item, value) - secret := reporting.Secret{ + secret := &reporting.Secret{ ID: itemId, Source: item.Source, RuleID: value.RuleID, @@ -76,7 +76,7 @@ func (s *Secrets) Detect(item plugins.Item, secretsChannel chan reporting.Secret EndColumn: value.EndColumn, Value: value.Secret, } - if !isSecretIgnored(&secret, &ignoredIds) { + if !isSecretIgnored(secret, &ignoredIds) { secretsChannel <- secret } else { log.Debug().Msgf("Secret %s was ignored", secret.ID) diff --git a/secrets/secrets_test.go b/secrets/secrets_test.go index 44bdccb6..31ba5fdc 100644 --- a/secrets/secrets_test.go +++ b/secrets/secrets_test.go @@ -123,7 +123,7 @@ func TestSecrets(t *testing.T) { } t.Run(name, func(t *testing.T) { fmt.Printf("Start test %s", name) - secretsChan := make(chan reporting.Secret, 1) + secretsChan := make(chan *reporting.Secret, 1) wg := &sync.WaitGroup{} wg.Add(1) detector.Detect(plugins.Item{Content: secret.Content}, secretsChan, wg, nil) From 3bc1589ff71829a4c677b2c3422c80157d22f752 Mon Sep 17 00:00:00 2001 From: Baruch Odem Date: Wed, 14 Feb 2024 11:28:59 +0200 Subject: [PATCH 03/18] Refactor code to move item and secret processing to separate functions --- cmd/main.go | 22 ++-------------------- cmd/workers.go | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 20 deletions(-) create mode 100644 cmd/workers.go diff --git a/cmd/main.go b/cmd/main.go index 9edacb44..242f2cb9 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -133,28 +133,10 @@ func preRun(cmd *cobra.Command, args []string) error { } channels.WaitGroup.Add(1) - go func() { - defer channels.WaitGroup.Done() - - wgItems := &sync.WaitGroup{} - for item := range channels.Items { - report.TotalItemsScanned++ - wgItems.Add(1) - go secrets.Detect(item, secretsChan, wgItems, ignoreVar) - } - wgItems.Wait() - close(secretsChan) - }() + go processItems(secrets) channels.WaitGroup.Add(1) - go func() { - defer channels.WaitGroup.Done() - for secret := range secretsChan { - report.TotalSecretsFound++ - report.Results[secret.ID] = append(report.Results[secret.ID], secret) - - } - }() + go processSecrets() return nil } diff --git a/cmd/workers.go b/cmd/workers.go new file mode 100644 index 00000000..f0947e01 --- /dev/null +++ b/cmd/workers.go @@ -0,0 +1,34 @@ +package cmd + +import ( + "sync" + + "github.com/checkmarx/2ms/secrets" +) + +func processItems(detector *secrets.Secrets) { + defer channels.WaitGroup.Done() + + wgItems := &sync.WaitGroup{} + for item := range channels.Items { + report.TotalItemsScanned++ + wgItems.Add(1) + go detector.Detect(item, secretsChan, wgItems, ignoreVar) + } + wgItems.Wait() + close(secretsChan) +} + +func processSecrets() { + defer channels.WaitGroup.Done() + + for secret := range secretsChan { + report.TotalSecretsFound++ + if validateVar { + validationChan <- secret + } + report.Results[secret.ID] = append(report.Results[secret.ID], secret) + } + close(validationChan) +} + From 8f0343e70a630e27bac82aa57cb2a37867d0a8fb Mon Sep 17 00:00:00 2001 From: Baruch Odem Date: Wed, 14 Feb 2024 14:57:06 +0200 Subject: [PATCH 04/18] Move Secret into secrets package --- reporting/report.go | 20 +++++--------------- reporting/report_test.go | 6 ++++-- reporting/sarif.go | 3 ++- secrets/engine.go | 7 +++---- secrets/engine_test.go | 3 +-- secrets/secret.go | 12 ++++++++++++ 6 files changed, 27 insertions(+), 24 deletions(-) create mode 100644 secrets/secret.go diff --git a/reporting/report.go b/reporting/report.go index 1be8437e..d57d42e3 100644 --- a/reporting/report.go +++ b/reporting/report.go @@ -6,6 +6,7 @@ import ( "strings" "github.com/checkmarx/2ms/config" + "github.com/checkmarx/2ms/secrets" "github.com/rs/zerolog/log" ) @@ -17,25 +18,14 @@ const ( ) type Report struct { - TotalItemsScanned int `json:"totalItemsScanned"` - TotalSecretsFound int `json:"totalSecretsFound"` - Results map[string][]*Secret `json:"results"` -} - -type Secret struct { - ID string `json:"id"` - Source string `json:"source"` - RuleID string `json:"ruleId"` - StartLine int `json:"startLine"` - EndLine int `json:"endLine"` - StartColumn int `json:"startColumn"` - EndColumn int `json:"endColumn"` - Value string `json:"value"` + TotalItemsScanned int `json:"totalItemsScanned"` + TotalSecretsFound int `json:"totalSecretsFound"` + Results map[string][]*secrets.Secret `json:"results"` } func Init() *Report { return &Report{ - Results: make(map[string][]*Secret), + Results: make(map[string][]*secrets.Secret), } } diff --git a/reporting/report_test.go b/reporting/report_test.go index 27576b4d..b028ed15 100644 --- a/reporting/report_test.go +++ b/reporting/report_test.go @@ -3,6 +3,8 @@ package reporting import ( "reflect" "testing" + + "github.com/checkmarx/2ms/secrets" ) func TestAddSecretToFile(t *testing.T) { @@ -23,9 +25,9 @@ JPcHeO7M6FohKgcEHX84koQDN98J/L7pFlSoU7WOl6f8BKavIdeSTPS9qQYWdQuT 4Xgur9w/aLZrLM3DSatR+kL+cVTyDTtgCt9Dc8k48Q== -----END RSA PRIVATE KEY-----`) - results := map[string][]*Secret{} + results := map[string][]*secrets.Secret{} report := Report{len(results), 1, results} - secret := &Secret{Source: "bla", StartLine: 0, StartColumn: 0, EndLine: 0, EndColumn: 0, Value: secretValue} + secret := &secrets.Secret{Source: "bla", StartLine: 0, StartColumn: 0, EndLine: 0, EndColumn: 0, Value: secretValue} source := "directory\\rawStringAsFile.txt" report.Results[source] = append(report.Results[source], secret) diff --git a/reporting/sarif.go b/reporting/sarif.go index db2f13f3..dc90f8a1 100644 --- a/reporting/sarif.go +++ b/reporting/sarif.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/checkmarx/2ms/config" + "github.com/checkmarx/2ms/secrets" ) func writeSarif(report Report, cfg *config.Config) (string, error) { @@ -74,7 +75,7 @@ func getResults(report Report) []Results { return results } -func getLocation(secret *Secret) []Locations { +func getLocation(secret *secrets.Secret) []Locations { return []Locations{ { PhysicalLocation: PhysicalLocation{ diff --git a/secrets/engine.go b/secrets/engine.go index 4f2f05a2..14e3568e 100644 --- a/secrets/engine.go +++ b/secrets/engine.go @@ -10,7 +10,6 @@ import ( "text/tabwriter" "github.com/checkmarx/2ms/plugins" - "github.com/checkmarx/2ms/reporting" "github.com/checkmarx/2ms/secrets/rules" "github.com/rs/zerolog/log" "github.com/spf13/cobra" @@ -58,7 +57,7 @@ func Init(secretsConfig SecretsConfig) (*Engine, error) { }, nil } -func (s *Engine) Detect(item plugins.Item, secretsChannel chan *reporting.Secret, wg *sync.WaitGroup, ignoredIds []string) { +func (s *Engine) Detect(item plugins.Item, secretsChannel chan *Secret, wg *sync.WaitGroup, ignoredIds []string) { defer wg.Done() fragment := detect.Fragment{ @@ -66,7 +65,7 @@ func (s *Engine) Detect(item plugins.Item, secretsChannel chan *reporting.Secret } for _, value := range s.detector.Detect(fragment) { itemId := getFindingId(item, value) - secret := &reporting.Secret{ + secret := &Secret{ ID: itemId, Source: item.Source, RuleID: value.RuleID, @@ -107,7 +106,7 @@ func getFindingId(item plugins.Item, finding report.Finding) string { return fmt.Sprintf("%x", sha) } -func isSecretIgnored(secret *reporting.Secret, ignoredIds *[]string) bool { +func isSecretIgnored(secret *Secret, ignoredIds *[]string) bool { for _, ignoredId := range *ignoredIds { if secret.ID == ignoredId { return true diff --git a/secrets/engine_test.go b/secrets/engine_test.go index 3df10a8c..a77964c1 100644 --- a/secrets/engine_test.go +++ b/secrets/engine_test.go @@ -6,7 +6,6 @@ import ( "testing" "github.com/checkmarx/2ms/plugins" - "github.com/checkmarx/2ms/reporting" "github.com/checkmarx/2ms/secrets/rules" ) @@ -123,7 +122,7 @@ func TestSecrets(t *testing.T) { } t.Run(name, func(t *testing.T) { fmt.Printf("Start test %s", name) - secretsChan := make(chan *reporting.Secret, 1) + secretsChan := make(chan *Secret, 1) wg := &sync.WaitGroup{} wg.Add(1) detector.Detect(plugins.Item{Content: secret.Content}, secretsChan, wg, nil) diff --git a/secrets/secret.go b/secrets/secret.go new file mode 100644 index 00000000..ea03a15f --- /dev/null +++ b/secrets/secret.go @@ -0,0 +1,12 @@ +package secrets + +type Secret struct { + ID string `json:"id"` + Source string `json:"source"` + RuleID string `json:"ruleId"` + StartLine int `json:"startLine"` + EndLine int `json:"endLine"` + StartColumn int `json:"startColumn"` + EndColumn int `json:"endColumn"` + Value string `json:"value"` +} From c62ba5e151a1644adefea7343316196ff02623d3 Mon Sep 17 00:00:00 2001 From: Baruch Odem Date: Thu, 15 Feb 2024 09:07:53 +0200 Subject: [PATCH 05/18] use go-yaml/yaml v3 For #203 --- go.mod | 3 +-- go.sum | 2 -- reporting/yaml.go | 2 +- 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 5ce41fac..4274bb5c 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/stretchr/testify v1.8.1 github.com/zricethezav/gitleaks/v8 v8.18.0 golang.org/x/time v0.1.0 - gopkg.in/yaml.v2 v2.4.0 + gopkg.in/yaml.v3 v3.0.1 ) require ( @@ -48,5 +48,4 @@ require ( golang.org/x/sys v0.15.0 // indirect golang.org/x/text v0.14.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 92e019ae..e885b7b6 100644 --- a/go.sum +++ b/go.sum @@ -533,8 +533,6 @@ gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/reporting/yaml.go b/reporting/yaml.go index e9335002..ad8e3d48 100644 --- a/reporting/yaml.go +++ b/reporting/yaml.go @@ -1,7 +1,7 @@ package reporting import ( - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" ) func writeYaml(report Report) (string, error) { From c6554be9ed6dd2659e08e2e109f20cbabe4e087c Mon Sep 17 00:00:00 2001 From: Baruch Odem Date: Thu, 15 Feb 2024 10:59:01 +0200 Subject: [PATCH 06/18] infrastructure for validity check --- README.md | 15 +++++++++++++++ cmd/main.go | 8 +++++++- cmd/workers.go | 15 +++++++++++++++ secrets/secret.go | 46 ++++++++++++++++++++++++++++++++++++++-------- 4 files changed, 75 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 62ea26fc..9275237c 100644 --- a/README.md +++ b/README.md @@ -151,6 +151,7 @@ Flags: --report-path strings path to generate report files. The output format will be determined by the file extension (.json, .yaml, .sarif) --rule strings select rules by name or tag to apply to this scan --stdout-format string stdout output format, available formats are: json, yaml, sarif (default "yaml") + --validate Validate the secrets found -v, --version version for 2ms Use "2ms [command] --help" for more information about a command. @@ -158,6 +159,20 @@ Use "2ms [command] --help" for more information about a command. +## Validity Check + +From the help message: `--validate Validate the secrets found`. + +The `--validate` flag will check the validity of the secrets found. For example, if it is a Github token, it will check if the token is valid by making a request to the Github API. We will use the less intrusive method to check the validity of the secret. + +The result of the validation can be: + +- `valid` - The secret is valid +- `revoked` - The secret is revoked +- `unknown` - We are not checking the validity of the secret + +If the `--validate` flag is not provided, the validation field will be omitted from the output, or its value will be an empty string. + ## Special Rules Special rules are rules that are not part of the default ruleset, usually because they are too noisy or too specific. You can use the `--add-special-rule` flag to add special rules by rule ID. diff --git a/cmd/main.go b/cmd/main.go index 7fff5127..fc3fee1f 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -72,7 +72,8 @@ var channels = plugins.Channels{ } var report = reporting.Init() -var secretsChan = make(chan *reporting.Secret) +var secretsChan = make(chan *secrets.Secret) +var validationChan = make(chan *secrets.Secret) func Execute() (int, error) { vConfig.SetEnvPrefix(envPrefix) @@ -138,6 +139,11 @@ func preRun(cmd *cobra.Command, args []string) error { channels.WaitGroup.Add(1) go processSecrets() + if validateVar { + channels.WaitGroup.Add(1) + go processValidation() + } + return nil } diff --git a/cmd/workers.go b/cmd/workers.go index 2246949b..2ebc5330 100644 --- a/cmd/workers.go +++ b/cmd/workers.go @@ -24,6 +24,21 @@ func processSecrets() { for secret := range secretsChan { report.TotalSecretsFound++ + if validateVar { + validationChan <- secret + } report.Results[secret.ID] = append(report.Results[secret.ID], secret) } + close(validationChan) +} + +func processValidation() { + defer channels.WaitGroup.Done() + + wgValidation := &sync.WaitGroup{} + for secret := range validationChan { + wgValidation.Add(1) + go secret.Validate(wgValidation) + } + wgValidation.Wait() } diff --git a/secrets/secret.go b/secrets/secret.go index ea03a15f..54959d0b 100644 --- a/secrets/secret.go +++ b/secrets/secret.go @@ -1,12 +1,42 @@ package secrets +import "sync" + +type ValidationResult string + +const ( + Valid ValidationResult = "Valid" + Revoked ValidationResult = "Revoked" + Unknown ValidationResult = "Unknown" +) + type Secret struct { - ID string `json:"id"` - Source string `json:"source"` - RuleID string `json:"ruleId"` - StartLine int `json:"startLine"` - EndLine int `json:"endLine"` - StartColumn int `json:"startColumn"` - EndColumn int `json:"endColumn"` - Value string `json:"value"` + ID string `json:"id"` + Source string `json:"source"` + RuleID string `json:"ruleId"` + StartLine int `json:"startLine"` + EndLine int `json:"endLine"` + StartColumn int `json:"startColumn"` + EndColumn int `json:"endColumn"` + Value string `json:"value"` + Validation ValidationResult `json:"validation,omitempty"` +} + +type validationFunc = func(*Secret) ValidationResult + +var ruleIDToFunction = map[string]validationFunc{ + "GitHub": validateGithub, +} + +func (s *Secret) Validate(wg *sync.WaitGroup) { + defer wg.Done() + if f, ok := ruleIDToFunction[s.RuleID]; ok { + s.Validation = f(s) + } else { + s.Validation = Unknown + } +} + +func validateGithub(s *Secret) ValidationResult { + return Unknown } From 5dbb9e9551cec2f0c0507a87f95d71e4ffc1cb41 Mon Sep 17 00:00:00 2001 From: Baruch Odem Date: Thu, 15 Feb 2024 11:22:10 +0200 Subject: [PATCH 07/18] add logic to include fine-grained PAT validation for GitHub --- README.md | 2 +- secrets/secret.go | 32 +++++++++++++++++++++++++++++--- 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 9275237c..96aa5e9e 100644 --- a/README.md +++ b/README.md @@ -169,7 +169,7 @@ The result of the validation can be: - `valid` - The secret is valid - `revoked` - The secret is revoked -- `unknown` - We are not checking the validity of the secret +- `unknown` - We failed to check, or we are not checking the validity of the secret at all If the `--validate` flag is not provided, the validation field will be omitted from the output, or its value will be an empty string. diff --git a/secrets/secret.go b/secrets/secret.go index 54959d0b..9aa01927 100644 --- a/secrets/secret.go +++ b/secrets/secret.go @@ -1,6 +1,12 @@ package secrets -import "sync" +import ( + "fmt" + "net/http" + "sync" + + "github.com/rs/zerolog/log" +) type ValidationResult string @@ -25,7 +31,8 @@ type Secret struct { type validationFunc = func(*Secret) ValidationResult var ruleIDToFunction = map[string]validationFunc{ - "GitHub": validateGithub, + "github-fine-grained-pat": validateGithub, + "github-pat": validateGithub, } func (s *Secret) Validate(wg *sync.WaitGroup) { @@ -38,5 +45,24 @@ func (s *Secret) Validate(wg *sync.WaitGroup) { } func validateGithub(s *Secret) ValidationResult { - return Unknown + const githubURL = "https://api.github.com/" + + req, err := http.NewRequest("GET", githubURL, nil) + if err != nil { + log.Warn().Err(err).Msg("Failed to validate secret") + return Unknown + } + req.Header.Set("Authorization", fmt.Sprintf("token %s", s.Value)) + + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + log.Warn().Err(err).Msg("Failed to validate secret") + return Unknown + } + + if resp.StatusCode == http.StatusOK { + return Valid + } + return Revoked } From 5c1452c45d1a3ecd31951d511d18e3924436b8e8 Mon Sep 17 00:00:00 2001 From: Baruch Odem Date: Sun, 18 Feb 2024 10:57:40 +0200 Subject: [PATCH 08/18] pairs --- secrets/pairs.go | 19 +++++++++++++++++++ secrets/secret.go | 8 ++++++++ 2 files changed, 27 insertions(+) create mode 100644 secrets/pairs.go diff --git a/secrets/pairs.go b/secrets/pairs.go new file mode 100644 index 00000000..94074797 --- /dev/null +++ b/secrets/pairs.go @@ -0,0 +1,19 @@ +package secrets + +var pairRules = [][]string{ + {"alibaba-access-key-id", "alibaba-secret-key"}, +} + +func generateAllKeys() map[string]bool { + allPaired := make(map[string]bool) + for _, pair := range pairRules { + for _, key := range pair { + allPaired[key] = true + } + } + return allPaired +} + +var allPaired = generateAllKeys() + +var pairedSecrets = make(map[string][]*Secret) diff --git a/secrets/secret.go b/secrets/secret.go index 9aa01927..33878e9e 100644 --- a/secrets/secret.go +++ b/secrets/secret.go @@ -33,12 +33,15 @@ type validationFunc = func(*Secret) ValidationResult var ruleIDToFunction = map[string]validationFunc{ "github-fine-grained-pat": validateGithub, "github-pat": validateGithub, + "alibaba": validateAlibaba, } func (s *Secret) Validate(wg *sync.WaitGroup) { defer wg.Done() if f, ok := ruleIDToFunction[s.RuleID]; ok { s.Validation = f(s) + } else if allPaired[s.RuleID] { + pairedSecrets[s.Source] = append(pairedSecrets[s.Source], s) } else { s.Validation = Unknown } @@ -66,3 +69,8 @@ func validateGithub(s *Secret) ValidationResult { } return Revoked } + +func validateAlibaba(s *Secret) ValidationResult { + // https://www.alibabacloud.com/help/en/sls/developer-reference/accesskey-pair + return Unknown +} From f5f36f552ba0d196ef60ed71d2df5c4b3673f1be Mon Sep 17 00:00:00 2001 From: Baruch Odem Date: Sun, 18 Feb 2024 15:39:14 +0200 Subject: [PATCH 09/18] rename SecretConfig to EngineConfig --- cmd/main.go | 2 +- secrets/engine.go | 6 +++--- secrets/engine_test.go | 10 +++++----- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index fc3fee1f..fb0a64b7 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -40,7 +40,7 @@ var ( customRegexRuleVar []string ignoreVar []string ignoreOnExitVar = ignoreOnExitNone - secretsConfigVar secrets.SecretsConfig + secretsConfigVar secrets.EngineConfig validateVar bool ) diff --git a/secrets/engine.go b/secrets/engine.go index 14e3568e..472829d6 100644 --- a/secrets/engine.go +++ b/secrets/engine.go @@ -25,7 +25,7 @@ type Engine struct { const customRegexRuleIdFormat = "custom-regex-%d" -type SecretsConfig struct { +type EngineConfig struct { SelectedList []string IgnoreList []string SpecialList []string @@ -33,7 +33,7 @@ type SecretsConfig struct { MaxTargetMegabytes int } -func Init(secretsConfig SecretsConfig) (*Engine, error) { +func Init(secretsConfig EngineConfig) (*Engine, error) { selectedRules := rules.FilterRules(secretsConfig.SelectedList, secretsConfig.IgnoreList, secretsConfig.SpecialList) if len(*selectedRules) == 0 { return nil, fmt.Errorf("no rules were selected") @@ -115,7 +115,7 @@ func isSecretIgnored(secret *Secret, ignoredIds *[]string) bool { return false } -func GetRulesCommand(secretsConfig *SecretsConfig) *cobra.Command { +func GetRulesCommand(secretsConfig *EngineConfig) *cobra.Command { return &cobra.Command{ Use: "rules", Short: "List all rules", diff --git a/secrets/engine_test.go b/secrets/engine_test.go index a77964c1..3336a0c7 100644 --- a/secrets/engine_test.go +++ b/secrets/engine_test.go @@ -15,12 +15,12 @@ func Test_Init(t *testing.T) { tests := []struct { name string - secretsConfig SecretsConfig + secretsConfig EngineConfig expectedErr error }{ { name: "selected and ignore flags used together for the same rule", - secretsConfig: SecretsConfig{ + secretsConfig: EngineConfig{ SelectedList: []string{allRules[0].Rule.RuleID}, IgnoreList: []string{allRules[0].Rule.RuleID}, SpecialList: []string{}, @@ -29,7 +29,7 @@ func Test_Init(t *testing.T) { }, { name: "non existent select flag", - secretsConfig: SecretsConfig{ + secretsConfig: EngineConfig{ SelectedList: []string{"non-existent-tag-name"}, IgnoreList: []string{}, SpecialList: []string{"non-existent-tag-name"}, @@ -38,7 +38,7 @@ func Test_Init(t *testing.T) { }, { name: "exiting special rule", - secretsConfig: SecretsConfig{ + secretsConfig: EngineConfig{ SelectedList: []string{"non-existent-tag-name"}, IgnoreList: []string{}, SpecialList: []string{specialRule.RuleID}, @@ -110,7 +110,7 @@ func TestSecrets(t *testing.T) { }, } - detector, err := Init(SecretsConfig{}) + detector, err := Init(EngineConfig{}) if err != nil { t.Fatal(err) } From 6a4ea7ad8127ab8c3721401ec90faffac4504508 Mon Sep 17 00:00:00 2001 From: Baruch Odem Date: Mon, 19 Feb 2024 16:48:19 +0200 Subject: [PATCH 10/18] alibaba pair validation --- cmd/main.go | 2 +- cmd/workers.go | 6 ++- secrets/alibaba.go | 91 ++++++++++++++++++++++++++++++++++++++++++++ secrets/engine.go | 19 +++++++-- secrets/pairs.go | 67 ++++++++++++++++++++++++++------ secrets/secret.go | 63 ------------------------------ secrets/validator.go | 74 +++++++++++++++++++++++++++++++++++ 7 files changed, 241 insertions(+), 81 deletions(-) create mode 100644 secrets/alibaba.go create mode 100644 secrets/validator.go diff --git a/cmd/main.go b/cmd/main.go index fb0a64b7..b91eff38 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -141,7 +141,7 @@ func preRun(cmd *cobra.Command, args []string) error { if validateVar { channels.WaitGroup.Add(1) - go processValidation() + go processValidation(engine) } return nil diff --git a/cmd/workers.go b/cmd/workers.go index 2ebc5330..029237c1 100644 --- a/cmd/workers.go +++ b/cmd/workers.go @@ -32,13 +32,15 @@ func processSecrets() { close(validationChan) } -func processValidation() { +func processValidation(engine *secrets.Engine) { defer channels.WaitGroup.Done() wgValidation := &sync.WaitGroup{} for secret := range validationChan { wgValidation.Add(1) - go secret.Validate(wgValidation) + go engine.RegisterForValidation(secret, wgValidation) } + engine.Validate() + wgValidation.Wait() } diff --git a/secrets/alibaba.go b/secrets/alibaba.go new file mode 100644 index 00000000..16688feb --- /dev/null +++ b/secrets/alibaba.go @@ -0,0 +1,91 @@ +package secrets + +import ( + "crypto/hmac" + "crypto/sha1" + "encoding/base64" + "io" + "net/http" + "net/url" + "strconv" + "strings" + "time" + + "github.com/rs/zerolog/log" +) + +// https://www.alibabacloud.com/help/en/sdk/alibaba-cloud-api-overview +// https://www.alibabacloud.com/help/en/sdk/product-overview/rpc-mechanism#sectiondiv-y9b-x9s-wvp + +func validateAlibaba(secrets pairsByRuleId) { + + accessKeys := secrets["alibaba-access-key-id"] + secretKeys := secrets["alibaba-secret-key"] + + for _, accessKey := range accessKeys { + for _, secretKey := range secretKeys { + statusCode, err := alibabaRequest(accessKey.Value, secretKey.Value) + if err != nil { + log.Warn().Err(err).Str("service", "alibaba").Msg("Failed to validate secret") + accessKey.Validation = Unknown + secretKey.Validation = Unknown + continue + } + + if statusCode == http.StatusOK { + accessKey.Validation = Valid + secretKey.Validation = Valid + } else { + accessKey.Validation = Revoked + secretKey.Validation = Revoked + } + } + } +} + +func alibabaRequest(accessKey, secretKey string) (int, error) { + req, err := http.NewRequest("GET", "https://ecs.aliyuncs.com/", nil) + if err != nil { + return 0, err + } + + // Workaround for gitleaks returns the key ends with " + accessKey = strings.TrimSuffix(accessKey, "\"") + secretKey = strings.TrimSuffix(secretKey, "\"") + + params := req.URL.Query() + params.Add("AccessKeyId", accessKey) + params.Add("Action", "DescribeRegions") + params.Add("SignatureMethod", "HMAC-SHA1") + params.Add("SignatureNonce", strconv.FormatInt(time.Now().UnixNano(), 10)) + params.Add("SignatureVersion", "1.0") + params.Add("Timestamp", time.Now().UTC().Format(time.RFC3339)) + params.Add("Version", "2014-05-26") + + stringToSign := "GET&%2F&" + url.QueryEscape(params.Encode()) + hmac := hmac.New(sha1.New, []byte(secretKey+"&")) + hmac.Write([]byte(stringToSign)) + signature := base64.StdEncoding.EncodeToString(hmac.Sum(nil)) + + params.Add("Signature", signature) + req.URL.RawQuery = params.Encode() + + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + return 0, err + } + + if resp.StatusCode != http.StatusOK { + defer resp.Body.Close() + body, _ := io.ReadAll(resp.Body) + log.Panic(). + Str("access_key", accessKey). + Str("service", "alibaba"). + Int("status_code", resp.StatusCode). + Msgf("Failed to validate secret %s", body) + } + + log.Debug().Str("service", "alibaba").Int("status_code", resp.StatusCode).Msg("Validated secret") + return resp.StatusCode, nil +} diff --git a/secrets/engine.go b/secrets/engine.go index 472829d6..71d727b3 100644 --- a/secrets/engine.go +++ b/secrets/engine.go @@ -19,8 +19,9 @@ import ( ) type Engine struct { - rules map[string]config.Rule - detector detect.Detector + rules map[string]config.Rule + detector detect.Detector + validator Validator } const customRegexRuleIdFormat = "custom-regex-%d" @@ -52,8 +53,9 @@ func Init(secretsConfig EngineConfig) (*Engine, error) { detector.MaxTargetMegaBytes = secretsConfig.MaxTargetMegabytes return &Engine{ - rules: rulesToBeApplied, - detector: *detector, + rules: rulesToBeApplied, + detector: *detector, + validator: *NewValidator(), }, nil } @@ -100,6 +102,15 @@ func (s *Engine) AddRegexRules(patterns []string) error { return nil } +func (s *Engine) RegisterForValidation(secret *Secret, wg *sync.WaitGroup) { + defer wg.Done() + s.validator.RegisterForValidation(secret) +} + +func (s *Engine) Validate() { + s.validator.Validate() +} + func getFindingId(item plugins.Item, finding report.Finding) string { idParts := []string{item.ID, finding.RuleID, finding.Secret} sha := sha1.Sum([]byte(strings.Join(idParts, "-"))) diff --git a/secrets/pairs.go b/secrets/pairs.go index 94074797..21be5c83 100644 --- a/secrets/pairs.go +++ b/secrets/pairs.go @@ -1,19 +1,64 @@ package secrets -var pairRules = [][]string{ - {"alibaba-access-key-id", "alibaba-secret-key"}, +import ( + "sync" +) + +type pairsByRuleId map[string][]*Secret +type pairsBySource map[string]pairsByRuleId +type pairsByGeneralKey map[string]pairsBySource + +type pairsCollector struct { + pairs pairsByGeneralKey } -func generateAllKeys() map[string]bool { - allPaired := make(map[string]bool) - for _, pair := range pairRules { - for _, key := range pair { - allPaired[key] = true - } +func newPairsCollector() *pairsCollector { + return &pairsCollector{pairs: make(pairsByGeneralKey)} +} + +func (p *pairsCollector) addIfNeeded(secret *Secret) bool { + generalKey, ok := ruleToGeneralKey[secret.RuleID] + if !ok { + return false } - return allPaired + + if _, ok := p.pairs[generalKey]; !ok { + p.pairs[generalKey] = make(pairsBySource) + } + if _, ok := p.pairs[generalKey][secret.Source]; !ok { + p.pairs[generalKey][secret.Source] = make(pairsByRuleId) + } + if _, ok := p.pairs[generalKey][secret.Source][secret.RuleID]; !ok { + p.pairs[generalKey][secret.Source][secret.RuleID] = make([]*Secret, 0) + } + + p.pairs[generalKey][secret.Source][secret.RuleID] = append(p.pairs[generalKey][secret.Source][secret.RuleID], secret) + return true } -var allPaired = generateAllKeys() +func (p *pairsCollector) validate(generalKey string, rulesById pairsByRuleId, wg *sync.WaitGroup) { + defer wg.Done() + generalKeyToValidation[generalKey](rulesById) +} + +type pairsValidationFunc func(pairsByRuleId) + +var generalKeyToValidation = map[string]pairsValidationFunc{ + "alibaba": validateAlibaba, +} + +var generalKeyToRules = map[string][]string{ + "alibaba": {"alibaba-access-key-id", "alibaba-secret-key"}, +} + +func generateRuleToGeneralKey() map[string]string { + ruleToGeneralKey := make(map[string]string) + for key, rules := range generalKeyToRules { + for _, rule := range rules { + ruleToGeneralKey[rule] = key + } + } + return ruleToGeneralKey +} -var pairedSecrets = make(map[string][]*Secret) +var ruleToGeneralKey = generateRuleToGeneralKey() diff --git a/secrets/secret.go b/secrets/secret.go index 33878e9e..c71167d0 100644 --- a/secrets/secret.go +++ b/secrets/secret.go @@ -1,21 +1,5 @@ package secrets -import ( - "fmt" - "net/http" - "sync" - - "github.com/rs/zerolog/log" -) - -type ValidationResult string - -const ( - Valid ValidationResult = "Valid" - Revoked ValidationResult = "Revoked" - Unknown ValidationResult = "Unknown" -) - type Secret struct { ID string `json:"id"` Source string `json:"source"` @@ -27,50 +11,3 @@ type Secret struct { Value string `json:"value"` Validation ValidationResult `json:"validation,omitempty"` } - -type validationFunc = func(*Secret) ValidationResult - -var ruleIDToFunction = map[string]validationFunc{ - "github-fine-grained-pat": validateGithub, - "github-pat": validateGithub, - "alibaba": validateAlibaba, -} - -func (s *Secret) Validate(wg *sync.WaitGroup) { - defer wg.Done() - if f, ok := ruleIDToFunction[s.RuleID]; ok { - s.Validation = f(s) - } else if allPaired[s.RuleID] { - pairedSecrets[s.Source] = append(pairedSecrets[s.Source], s) - } else { - s.Validation = Unknown - } -} - -func validateGithub(s *Secret) ValidationResult { - const githubURL = "https://api.github.com/" - - req, err := http.NewRequest("GET", githubURL, nil) - if err != nil { - log.Warn().Err(err).Msg("Failed to validate secret") - return Unknown - } - req.Header.Set("Authorization", fmt.Sprintf("token %s", s.Value)) - - client := &http.Client{} - resp, err := client.Do(req) - if err != nil { - log.Warn().Err(err).Msg("Failed to validate secret") - return Unknown - } - - if resp.StatusCode == http.StatusOK { - return Valid - } - return Revoked -} - -func validateAlibaba(s *Secret) ValidationResult { - // https://www.alibabacloud.com/help/en/sls/developer-reference/accesskey-pair - return Unknown -} diff --git a/secrets/validator.go b/secrets/validator.go new file mode 100644 index 00000000..7a189621 --- /dev/null +++ b/secrets/validator.go @@ -0,0 +1,74 @@ +package secrets + +import ( + "fmt" + "net/http" + "sync" + + "github.com/rs/zerolog/log" +) + +type ValidationResult string + +const ( + Valid ValidationResult = "Valid" + Revoked ValidationResult = "Revoked" + Unknown ValidationResult = "Unknown" +) + +type validationFunc = func(*Secret) ValidationResult + +var ruleIDToFunction = map[string]validationFunc{ + "github-fine-grained-pat": validateGithub, + "github-pat": validateGithub, +} + +func validateGithub(s *Secret) ValidationResult { + const githubURL = "https://api.github.com/" + + req, err := http.NewRequest("GET", githubURL, nil) + if err != nil { + log.Warn().Err(err).Msg("Failed to validate secret") + return Unknown + } + req.Header.Set("Authorization", fmt.Sprintf("token %s", s.Value)) + + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + log.Warn().Err(err).Msg("Failed to validate secret") + return Unknown + } + + if resp.StatusCode == http.StatusOK { + return Valid + } + return Revoked +} + +type Validator struct { + pairsCollector *pairsCollector +} + +func NewValidator() *Validator { + return &Validator{pairsCollector: newPairsCollector()} +} + +func (v *Validator) RegisterForValidation(secret *Secret) { + if validate, ok := ruleIDToFunction[secret.RuleID]; ok { + secret.Validation = validate(secret) + } else if !v.pairsCollector.addIfNeeded(secret) { + secret.Validation = Unknown + } +} + +func (v *Validator) Validate() { + wg := &sync.WaitGroup{} + for generalKey, bySource := range v.pairsCollector.pairs { + for _, byRule := range bySource { + // test all pairs per source + wg.Add(1) + v.pairsCollector.validate(generalKey, byRule, wg) + } + } +} From d94ee1bcf96dc06ab6b1427b98beaa0aec1c845a Mon Sep 17 00:00:00 2001 From: Baruch Odem Date: Mon, 19 Feb 2024 17:45:41 +0200 Subject: [PATCH 11/18] wait for all validation registrations --- cmd/workers.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/workers.go b/cmd/workers.go index 029237c1..4a6ff7dc 100644 --- a/cmd/workers.go +++ b/cmd/workers.go @@ -40,7 +40,7 @@ func processValidation(engine *secrets.Engine) { wgValidation.Add(1) go engine.RegisterForValidation(secret, wgValidation) } - engine.Validate() - wgValidation.Wait() + + engine.Validate() } From 84d9119f8ff80d296f751204ec3b40e4b7c009f0 Mon Sep 17 00:00:00 2001 From: Baruch Odem Date: Tue, 20 Feb 2024 09:05:20 +0200 Subject: [PATCH 12/18] add missing wait --- secrets/validator.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/secrets/validator.go b/secrets/validator.go index f3b79271..778963c0 100644 --- a/secrets/validator.go +++ b/secrets/validator.go @@ -66,9 +66,9 @@ func (v *Validator) Validate() { wg := &sync.WaitGroup{} for generalKey, bySource := range v.pairsCollector.pairs { for _, byRule := range bySource { - // test all pairs per source wg.Add(1) v.pairsCollector.validate(generalKey, byRule, wg) } } + wg.Wait() } From 55bf2a1fdd0e63311b14d27adea159805d56b112 Mon Sep 17 00:00:00 2001 From: Baruch Odem Date: Tue, 20 Feb 2024 09:41:27 +0200 Subject: [PATCH 13/18] Fix validationStatus type in secret.go and validator.go --- secrets/secret.go | 2 +- secrets/validator.go | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/secrets/secret.go b/secrets/secret.go index 5c52962a..38a613af 100644 --- a/secrets/secret.go +++ b/secrets/secret.go @@ -13,7 +13,7 @@ type Secret struct { StartColumn int `json:"startColumn"` EndColumn int `json:"endColumn"` Value string `json:"value"` - ValidationStatus ValidationResult `json:"validationStatus,omitempty"` + ValidationStatus validationResult `json:"validationStatus,omitempty"` } func isCanValidateRule(ruleID string) bool { diff --git a/secrets/validator.go b/secrets/validator.go index 778963c0..54e1b53c 100644 --- a/secrets/validator.go +++ b/secrets/validator.go @@ -8,22 +8,22 @@ import ( "github.com/rs/zerolog/log" ) -type ValidationResult string +type validationResult string const ( - Valid ValidationResult = "Valid" - Revoked ValidationResult = "Revoked" - Unknown ValidationResult = "Unknown" + Valid validationResult = "Valid" + Revoked validationResult = "Revoked" + Unknown validationResult = "Unknown" ) -type validationFunc = func(*Secret) ValidationResult +type validationFunc = func(*Secret) validationResult var ruleIDToFunction = map[string]validationFunc{ "github-fine-grained-pat": validateGithub, "github-pat": validateGithub, } -func validateGithub(s *Secret) ValidationResult { +func validateGithub(s *Secret) validationResult { const githubURL = "https://api.github.com/" req, err := http.NewRequest("GET", githubURL, nil) From b23e4a6638ff9b3b138f899d7ed462a77c772174 Mon Sep 17 00:00:00 2001 From: Baruch Odem Date: Tue, 20 Feb 2024 09:41:35 +0200 Subject: [PATCH 14/18] Refactor Alibaba secret validation logic --- secrets/alibaba.go | 43 ++++++++++++++++++------------------------- 1 file changed, 18 insertions(+), 25 deletions(-) diff --git a/secrets/alibaba.go b/secrets/alibaba.go index 00aef41b..93038d61 100644 --- a/secrets/alibaba.go +++ b/secrets/alibaba.go @@ -4,7 +4,7 @@ import ( "crypto/hmac" "crypto/sha1" "encoding/base64" - "io" + "fmt" "net/http" "net/url" "strconv" @@ -24,29 +24,21 @@ func validateAlibaba(secrets pairsByRuleId) { for _, accessKey := range accessKeys { for _, secretKey := range secretKeys { - statusCode, err := alibabaRequest(accessKey.Value, secretKey.Value) + status, err := alibabaRequest(accessKey.Value, secretKey.Value) if err != nil { log.Warn().Err(err).Str("service", "alibaba").Msg("Failed to validate secret") - accessKey.ValidationStatus = Unknown - secretKey.ValidationStatus = Unknown - continue } + accessKey.ValidationStatus = status + secretKey.ValidationStatus = status - if statusCode == http.StatusOK { - accessKey.ValidationStatus = Valid - secretKey.ValidationStatus = Valid - } else { - accessKey.ValidationStatus = Revoked - secretKey.ValidationStatus = Revoked - } } } } -func alibabaRequest(accessKey, secretKey string) (int, error) { +func alibabaRequest(accessKey, secretKey string) (validationResult, error) { req, err := http.NewRequest("GET", "https://ecs.aliyuncs.com/", nil) if err != nil { - return 0, err + return Unknown, err } // Workaround for gitleaks returns the key ends with " @@ -73,19 +65,20 @@ func alibabaRequest(accessKey, secretKey string) (int, error) { client := &http.Client{} resp, err := client.Do(req) if err != nil { - return 0, err + return Unknown, err + } + log.Debug().Str("service", "alibaba").Int("status_code", resp.StatusCode) + + // If the access key is invalid, the response will be 404 + // If the secret key is invalid, the response will be 400 along with other signautre Errors + if resp.StatusCode == http.StatusNotFound || resp.StatusCode == http.StatusBadRequest { + return Revoked, nil } - if resp.StatusCode != http.StatusOK { - defer resp.Body.Close() - body, _ := io.ReadAll(resp.Body) - log.Panic(). - Str("access_key", accessKey). - Str("service", "alibaba"). - Int("status_code", resp.StatusCode). - Msgf("Failed to validate secret %s", body) + if resp.StatusCode == http.StatusOK { + return Valid, nil } - log.Debug().Str("service", "alibaba").Int("status_code", resp.StatusCode).Msg("Validated secret") - return resp.StatusCode, nil + err = fmt.Errorf("unexpected status code: %d", resp.StatusCode) + return Unknown, err } From 4a48d5acff0b10b39354a03ccdb7039e818ebeb7 Mon Sep 17 00:00:00 2001 From: Baruch Odem Date: Tue, 20 Feb 2024 09:43:41 +0200 Subject: [PATCH 15/18] todo --- secrets/secret.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/secrets/secret.go b/secrets/secret.go index 38a613af..b0c95819 100644 --- a/secrets/secret.go +++ b/secrets/secret.go @@ -1,5 +1,8 @@ package secrets +// TODO: rename package to engine and move secrets into subpackage +// Then move the validators into a subpackage too + import ( "sync" ) From a1129ca3f018d126f61c6f0db5e6f4e39b88ff5d Mon Sep 17 00:00:00 2001 From: Baruch Odem Date: Tue, 20 Feb 2024 10:51:03 +0200 Subject: [PATCH 16/18] update accessKey validation if worst --- secrets/alibaba.go | 8 +++++-- secrets/validator.go | 24 +++++++++++++++++++ secrets/validator_test.go | 49 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 79 insertions(+), 2 deletions(-) create mode 100644 secrets/validator_test.go diff --git a/secrets/alibaba.go b/secrets/alibaba.go index 93038d61..136b0556 100644 --- a/secrets/alibaba.go +++ b/secrets/alibaba.go @@ -23,14 +23,18 @@ func validateAlibaba(secrets pairsByRuleId) { secretKeys := secrets["alibaba-secret-key"] for _, accessKey := range accessKeys { + accessKey.ValidationStatus = Unknown + for _, secretKey := range secretKeys { status, err := alibabaRequest(accessKey.Value, secretKey.Value) if err != nil { log.Warn().Err(err).Str("service", "alibaba").Msg("Failed to validate secret") } - accessKey.ValidationStatus = status - secretKey.ValidationStatus = status + secretKey.ValidationStatus = status + if accessKey.ValidationStatus.CompareTo(status) == second { + accessKey.ValidationStatus = status + } } } } diff --git a/secrets/validator.go b/secrets/validator.go index 54e1b53c..748cd640 100644 --- a/secrets/validator.go +++ b/secrets/validator.go @@ -16,6 +16,30 @@ const ( Unknown validationResult = "Unknown" ) +type compared int + +const ( + first compared = -1 + second compared = 1 + equal compared = 0 +) + +func (v validationResult) CompareTo(other validationResult) compared { + if v == other { + return equal + } + if v == Unknown { + return second + } + if other == Unknown { + return first + } + if v == Revoked { + return second + } + return first +} + type validationFunc = func(*Secret) validationResult var ruleIDToFunction = map[string]validationFunc{ diff --git a/secrets/validator_test.go b/secrets/validator_test.go new file mode 100644 index 00000000..e4d0a216 --- /dev/null +++ b/secrets/validator_test.go @@ -0,0 +1,49 @@ +package secrets + +import ( + "testing" +) + +func TestValidationResultCompareTo(t *testing.T) { + testCases := []struct { + first validationResult + second validationResult + want compared + message string + }{ + { + first: Valid, + second: Valid, + want: equal, + message: "Valid should be equal to Valid", + }, + { + first: Revoked, + second: Valid, + want: second, + message: "Valid should be greater than Revoked", + }, + { + first: Valid, + second: Unknown, + want: first, + message: "Valid should be greater than Unknown", + }, + { + first: Unknown, + second: Revoked, + want: second, + message: "Revoked should be greater than Unknown", + }, + } + + for _, tc := range testCases { + t.Run(tc.message, func(t *testing.T) { + got := tc.first.CompareTo(tc.second) + if got != tc.want { + t.Errorf("got %d, want %d", got, tc.want) + } + }, + ) + } +} From f364f89572cf4d434c17592e1b0cda365352b2ad Mon Sep 17 00:00:00 2001 From: Baruch Odem Date: Tue, 20 Feb 2024 10:54:53 +0200 Subject: [PATCH 17/18] update list of rules --- docs/list-of-rules.md | 4 ++-- secrets/secret.go | 21 +++++++-------------- 2 files changed, 9 insertions(+), 16 deletions(-) diff --git a/docs/list-of-rules.md b/docs/list-of-rules.md index b4a0f7b0..1ca56ae8 100644 --- a/docs/list-of-rules.md +++ b/docs/list-of-rules.md @@ -11,8 +11,8 @@ Here is a complete list of all the rules that are currently implemented. | age secret key | Age secret key | secret-key | | | airtable-api-key | Airtable API Key | api-key | | | algolia-api-key | Algolia API Key | api-key | | -| alibaba-access-key-id | Alibaba AccessKey ID | access-key,access-id | | -| alibaba-secret-key | Alibaba Secret Key | secret-key | | +| alibaba-access-key-id | Alibaba AccessKey ID | access-key,access-id | V | +| alibaba-secret-key | Alibaba Secret Key | secret-key | V | | asana-client-id | Asana Client ID | client-id | | | asana-client-secret | Asana Client Secret | client-secret | | | atlassian-api-token | Atlassian API token | api-token | | diff --git a/secrets/secret.go b/secrets/secret.go index b0c95819..e3290c40 100644 --- a/secrets/secret.go +++ b/secrets/secret.go @@ -3,10 +3,6 @@ package secrets // TODO: rename package to engine and move secrets into subpackage // Then move the validators into a subpackage too -import ( - "sync" -) - type Secret struct { ID string `json:"id"` Source string `json:"source"` @@ -20,15 +16,12 @@ type Secret struct { } func isCanValidateRule(ruleID string) bool { - _, ok := ruleIDToFunction[ruleID] - return ok -} - -func (s *Secret) Validate(wg *sync.WaitGroup) { - defer wg.Done() - if f, ok := ruleIDToFunction[s.RuleID]; ok { - s.ValidationStatus = f(s) - } else { - s.ValidationStatus = Unknown + if _, ok := ruleIDToFunction[ruleID]; ok { + return true } + if _, ok := ruleToGeneralKey[ruleID]; ok { + return true + } + + return false } From 5e96cd78e225c298f9adb51b07fdc9ec587dbb6d Mon Sep 17 00:00:00 2001 From: Baruch Odem Date: Tue, 20 Feb 2024 11:56:04 +0200 Subject: [PATCH 18/18] comment https://github.com/gitleaks/gitleaks/pull/1350 --- secrets/alibaba.go | 1 + 1 file changed, 1 insertion(+) diff --git a/secrets/alibaba.go b/secrets/alibaba.go index 136b0556..d81cfc45 100644 --- a/secrets/alibaba.go +++ b/secrets/alibaba.go @@ -46,6 +46,7 @@ func alibabaRequest(accessKey, secretKey string) (validationResult, error) { } // Workaround for gitleaks returns the key ends with " + // https://github.com/gitleaks/gitleaks/pull/1350 accessKey = strings.TrimSuffix(accessKey, "\"") secretKey = strings.TrimSuffix(secretKey, "\"")