diff --git a/cmd/main.go b/cmd/main.go index 4046e998..87b86ce1 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -141,7 +141,7 @@ func preRun(cmd *cobra.Command, args []string) error { if validateVar { channels.WaitGroup.Add(1) - go processValidation() + go processValidation(engine) } return nil diff --git a/cmd/workers.go b/cmd/workers.go index 2ebc5330..4a6ff7dc 100644 --- a/cmd/workers.go +++ b/cmd/workers.go @@ -32,13 +32,15 @@ func processSecrets() { close(validationChan) } -func processValidation() { +func processValidation(engine *secrets.Engine) { defer channels.WaitGroup.Done() wgValidation := &sync.WaitGroup{} for secret := range validationChan { wgValidation.Add(1) - go secret.Validate(wgValidation) + go engine.RegisterForValidation(secret, wgValidation) } wgValidation.Wait() + + engine.Validate() } diff --git a/docs/list-of-rules.md b/docs/list-of-rules.md index b4a0f7b0..1ca56ae8 100644 --- a/docs/list-of-rules.md +++ b/docs/list-of-rules.md @@ -11,8 +11,8 @@ Here is a complete list of all the rules that are currently implemented. | age secret key | Age secret key | secret-key | | | airtable-api-key | Airtable API Key | api-key | | | algolia-api-key | Algolia API Key | api-key | | -| alibaba-access-key-id | Alibaba AccessKey ID | access-key,access-id | | -| alibaba-secret-key | Alibaba Secret Key | secret-key | | +| alibaba-access-key-id | Alibaba AccessKey ID | access-key,access-id | V | +| alibaba-secret-key | Alibaba Secret Key | secret-key | V | | asana-client-id | Asana Client ID | client-id | | | asana-client-secret | Asana Client Secret | client-secret | | | atlassian-api-token | Atlassian API token | api-token | | diff --git a/secrets/alibaba.go b/secrets/alibaba.go new file mode 100644 index 00000000..d81cfc45 --- /dev/null +++ b/secrets/alibaba.go @@ -0,0 +1,89 @@ +package secrets + +import ( + "crypto/hmac" + "crypto/sha1" + "encoding/base64" + "fmt" + "net/http" + "net/url" + "strconv" + "strings" + "time" + + "github.com/rs/zerolog/log" +) + +// https://www.alibabacloud.com/help/en/sdk/alibaba-cloud-api-overview +// https://www.alibabacloud.com/help/en/sdk/product-overview/rpc-mechanism#sectiondiv-y9b-x9s-wvp + +func validateAlibaba(secrets pairsByRuleId) { + + accessKeys := secrets["alibaba-access-key-id"] + secretKeys := secrets["alibaba-secret-key"] + + for _, accessKey := range accessKeys { + accessKey.ValidationStatus = Unknown + + for _, secretKey := range secretKeys { + status, err := alibabaRequest(accessKey.Value, secretKey.Value) + if err != nil { + log.Warn().Err(err).Str("service", "alibaba").Msg("Failed to validate secret") + } + + secretKey.ValidationStatus = status + if accessKey.ValidationStatus.CompareTo(status) == second { + accessKey.ValidationStatus = status + } + } + } +} + +func alibabaRequest(accessKey, secretKey string) (validationResult, error) { + req, err := http.NewRequest("GET", "https://ecs.aliyuncs.com/", nil) + if err != nil { + return Unknown, err + } + + // Workaround for gitleaks returns the key ends with " + // https://github.com/gitleaks/gitleaks/pull/1350 + accessKey = strings.TrimSuffix(accessKey, "\"") + secretKey = strings.TrimSuffix(secretKey, "\"") + + params := req.URL.Query() + params.Add("AccessKeyId", accessKey) + params.Add("Action", "DescribeRegions") + params.Add("SignatureMethod", "HMAC-SHA1") + params.Add("SignatureNonce", strconv.FormatInt(time.Now().UnixNano(), 10)) + params.Add("SignatureVersion", "1.0") + params.Add("Timestamp", time.Now().UTC().Format(time.RFC3339)) + params.Add("Version", "2014-05-26") + + stringToSign := "GET&%2F&" + url.QueryEscape(params.Encode()) + hmac := hmac.New(sha1.New, []byte(secretKey+"&")) + hmac.Write([]byte(stringToSign)) + signature := base64.StdEncoding.EncodeToString(hmac.Sum(nil)) + + params.Add("Signature", signature) + req.URL.RawQuery = params.Encode() + + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + return Unknown, err + } + log.Debug().Str("service", "alibaba").Int("status_code", resp.StatusCode) + + // If the access key is invalid, the response will be 404 + // If the secret key is invalid, the response will be 400 along with other signautre Errors + if resp.StatusCode == http.StatusNotFound || resp.StatusCode == http.StatusBadRequest { + return Revoked, nil + } + + if resp.StatusCode == http.StatusOK { + return Valid, nil + } + + err = fmt.Errorf("unexpected status code: %d", resp.StatusCode) + return Unknown, err +} diff --git a/secrets/engine.go b/secrets/engine.go index 2aa331a0..5ec94385 100644 --- a/secrets/engine.go +++ b/secrets/engine.go @@ -19,8 +19,9 @@ import ( ) type Engine struct { - rules map[string]config.Rule - detector detect.Detector + rules map[string]config.Rule + detector detect.Detector + validator Validator } const customRegexRuleIdFormat = "custom-regex-%d" @@ -52,8 +53,9 @@ func Init(engineConfig EngineConfig) (*Engine, error) { detector.MaxTargetMegaBytes = engineConfig.MaxTargetMegabytes return &Engine{ - rules: rulesToBeApplied, - detector: *detector, + rules: rulesToBeApplied, + detector: *detector, + validator: *NewValidator(), }, nil } @@ -100,6 +102,15 @@ func (s *Engine) AddRegexRules(patterns []string) error { return nil } +func (s *Engine) RegisterForValidation(secret *Secret, wg *sync.WaitGroup) { + defer wg.Done() + s.validator.RegisterForValidation(secret) +} + +func (s *Engine) Validate() { + s.validator.Validate() +} + func getFindingId(item plugins.Item, finding report.Finding) string { idParts := []string{item.ID, finding.RuleID, finding.Secret} sha := sha1.Sum([]byte(strings.Join(idParts, "-"))) diff --git a/secrets/pairs.go b/secrets/pairs.go new file mode 100644 index 00000000..21be5c83 --- /dev/null +++ b/secrets/pairs.go @@ -0,0 +1,64 @@ +package secrets + +import ( + "sync" +) + +type pairsByRuleId map[string][]*Secret +type pairsBySource map[string]pairsByRuleId +type pairsByGeneralKey map[string]pairsBySource + +type pairsCollector struct { + pairs pairsByGeneralKey +} + +func newPairsCollector() *pairsCollector { + return &pairsCollector{pairs: make(pairsByGeneralKey)} +} + +func (p *pairsCollector) addIfNeeded(secret *Secret) bool { + generalKey, ok := ruleToGeneralKey[secret.RuleID] + if !ok { + return false + } + + if _, ok := p.pairs[generalKey]; !ok { + p.pairs[generalKey] = make(pairsBySource) + } + if _, ok := p.pairs[generalKey][secret.Source]; !ok { + p.pairs[generalKey][secret.Source] = make(pairsByRuleId) + } + if _, ok := p.pairs[generalKey][secret.Source][secret.RuleID]; !ok { + p.pairs[generalKey][secret.Source][secret.RuleID] = make([]*Secret, 0) + } + + p.pairs[generalKey][secret.Source][secret.RuleID] = append(p.pairs[generalKey][secret.Source][secret.RuleID], secret) + return true +} + +func (p *pairsCollector) validate(generalKey string, rulesById pairsByRuleId, wg *sync.WaitGroup) { + defer wg.Done() + generalKeyToValidation[generalKey](rulesById) +} + +type pairsValidationFunc func(pairsByRuleId) + +var generalKeyToValidation = map[string]pairsValidationFunc{ + "alibaba": validateAlibaba, +} + +var generalKeyToRules = map[string][]string{ + "alibaba": {"alibaba-access-key-id", "alibaba-secret-key"}, +} + +func generateRuleToGeneralKey() map[string]string { + ruleToGeneralKey := make(map[string]string) + for key, rules := range generalKeyToRules { + for _, rule := range rules { + ruleToGeneralKey[rule] = key + } + } + return ruleToGeneralKey +} + +var ruleToGeneralKey = generateRuleToGeneralKey() diff --git a/secrets/secret.go b/secrets/secret.go index 274653bc..e3290c40 100644 --- a/secrets/secret.go +++ b/secrets/secret.go @@ -1,20 +1,7 @@ package secrets -import ( - "fmt" - "net/http" - "sync" - - "github.com/rs/zerolog/log" -) - -type ValidationResult string - -const ( - Valid ValidationResult = "Valid" - Revoked ValidationResult = "Revoked" - Unknown ValidationResult = "Unknown" -) +// TODO: rename package to engine and move secrets into subpackage +// Then move the validators into a subpackage too type Secret struct { ID string `json:"id"` @@ -25,49 +12,16 @@ type Secret struct { StartColumn int `json:"startColumn"` EndColumn int `json:"endColumn"` Value string `json:"value"` - ValidationStatus ValidationResult `json:"validationStatus,omitempty"` -} - -type validationFunc = func(*Secret) ValidationResult - -var ruleIDToFunction = map[string]validationFunc{ - "github-fine-grained-pat": validateGithub, - "github-pat": validateGithub, + ValidationStatus validationResult `json:"validationStatus,omitempty"` } func isCanValidateRule(ruleID string) bool { - _, ok := ruleIDToFunction[ruleID] - return ok -} - -func (s *Secret) Validate(wg *sync.WaitGroup) { - defer wg.Done() - if f, ok := ruleIDToFunction[s.RuleID]; ok { - s.ValidationStatus = f(s) - } else { - s.ValidationStatus = Unknown + if _, ok := ruleIDToFunction[ruleID]; ok { + return true } -} - -func validateGithub(s *Secret) ValidationResult { - const githubURL = "https://api.github.com/" - - req, err := http.NewRequest("GET", githubURL, nil) - if err != nil { - log.Warn().Err(err).Msg("Failed to validate secret") - return Unknown + if _, ok := ruleToGeneralKey[ruleID]; ok { + return true } - req.Header.Set("Authorization", fmt.Sprintf("token %s", s.Value)) - client := &http.Client{} - resp, err := client.Do(req) - if err != nil { - log.Warn().Err(err).Msg("Failed to validate secret") - return Unknown - } - - if resp.StatusCode == http.StatusOK { - return Valid - } - return Revoked + return false } diff --git a/secrets/validator.go b/secrets/validator.go new file mode 100644 index 00000000..748cd640 --- /dev/null +++ b/secrets/validator.go @@ -0,0 +1,98 @@ +package secrets + +import ( + "fmt" + "net/http" + "sync" + + "github.com/rs/zerolog/log" +) + +type validationResult string + +const ( + Valid validationResult = "Valid" + Revoked validationResult = "Revoked" + Unknown validationResult = "Unknown" +) + +type compared int + +const ( + first compared = -1 + second compared = 1 + equal compared = 0 +) + +func (v validationResult) CompareTo(other validationResult) compared { + if v == other { + return equal + } + if v == Unknown { + return second + } + if other == Unknown { + return first + } + if v == Revoked { + return second + } + return first +} + +type validationFunc = func(*Secret) validationResult + +var ruleIDToFunction = map[string]validationFunc{ + "github-fine-grained-pat": validateGithub, + "github-pat": validateGithub, +} + +func validateGithub(s *Secret) validationResult { + const githubURL = "https://api.github.com/" + + req, err := http.NewRequest("GET", githubURL, nil) + if err != nil { + log.Warn().Err(err).Msg("Failed to validate secret") + return Unknown + } + req.Header.Set("Authorization", fmt.Sprintf("token %s", s.Value)) + + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + log.Warn().Err(err).Msg("Failed to validate secret") + return Unknown + } + + if resp.StatusCode == http.StatusOK { + return Valid + } + return Revoked +} + +type Validator struct { + pairsCollector *pairsCollector +} + +func NewValidator() *Validator { + return &Validator{pairsCollector: newPairsCollector()} +} + +func (v *Validator) RegisterForValidation(secret *Secret) { + if validate, ok := ruleIDToFunction[secret.RuleID]; ok { + secret.ValidationStatus = validate(secret) + } else if !v.pairsCollector.addIfNeeded(secret) { + secret.ValidationStatus = Unknown + } +} + +func (v *Validator) Validate() { + wg := &sync.WaitGroup{} + for generalKey, bySource := range v.pairsCollector.pairs { + for _, byRule := range bySource { + wg.Add(1) + v.pairsCollector.validate(generalKey, byRule, wg) + } + } + wg.Wait() +} diff --git a/secrets/validator_test.go b/secrets/validator_test.go new file mode 100644 index 00000000..e4d0a216 --- /dev/null +++ b/secrets/validator_test.go @@ -0,0 +1,49 @@ +package secrets + +import ( + "testing" +) + +func TestValidationResultCompareTo(t *testing.T) { + testCases := []struct { + first validationResult + second validationResult + want compared + message string + }{ + { + first: Valid, + second: Valid, + want: equal, + message: "Valid should be equal to Valid", + }, + { + first: Revoked, + second: Valid, + want: second, + message: "Valid should be greater than Revoked", + }, + { + first: Valid, + second: Unknown, + want: first, + message: "Valid should be greater than Unknown", + }, + { + first: Unknown, + second: Revoked, + want: second, + message: "Revoked should be greater than Unknown", + }, + } + + for _, tc := range testCases { + t.Run(tc.message, func(t *testing.T) { + got := tc.first.CompareTo(tc.second) + if got != tc.want { + t.Errorf("got %d, want %d", got, tc.want) + } + }, + ) + } +}