feat: validate pairs of secrets

cmd/main.go

@@ -40,7 +40,7 @@ var (
 	customRegexRuleVar []string
 	ignoreVar          []string
 	ignoreOnExitVar    = ignoreOnExitNone
-	secretsConfigVar   secrets.SecretsConfig
+	secretsConfigVar   secrets.EngineConfig
 	validateVar        bool
 )
 
@@ -141,7 +141,7 @@ func preRun(cmd *cobra.Command, args []string) error {
 
 	if validateVar {
 		channels.WaitGroup.Add(1)
-		go processValidation()
+		go processValidation(engine)
 	}
 
 	return nil

cmd/workers.go

@@ -32,13 +32,15 @@ func processSecrets() {
 	close(validationChan)
 }
 
-func processValidation() {
+func processValidation(engine *secrets.Engine) {
 	defer channels.WaitGroup.Done()
 
 	wgValidation := &sync.WaitGroup{}
 	for secret := range validationChan {
 		wgValidation.Add(1)
-		go secret.Validate(wgValidation)
+		go engine.RegisterForValidation(secret, wgValidation)
 	}
 	wgValidation.Wait()
+
+	engine.Validate()
 }

secrets/alibaba.go

@@ -0,0 +1,91 @@
+package secrets
+
+import (
+	"crypto/hmac"
+	"crypto/sha1"
+	"encoding/base64"
+	"io"
+	"net/http"
+	"net/url"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/rs/zerolog/log"
+)
+
+// https://www.alibabacloud.com/help/en/sdk/alibaba-cloud-api-overview
+// https://www.alibabacloud.com/help/en/sdk/product-overview/rpc-mechanism#sectiondiv-y9b-x9s-wvp
+
+func validateAlibaba(secrets pairsByRuleId) {
+
+	accessKeys := secrets["alibaba-access-key-id"]
+	secretKeys := secrets["alibaba-secret-key"]
+
+	for _, accessKey := range accessKeys {
+		for _, secretKey := range secretKeys {
+			statusCode, err := alibabaRequest(accessKey.Value, secretKey.Value)
+			if err != nil {
+				log.Warn().Err(err).Str("service", "alibaba").Msg("Failed to validate secret")
+				accessKey.ValidationStatus = Unknown
+				secretKey.ValidationStatus = Unknown
+				continue
+			}
+
+			if statusCode == http.StatusOK {
+				accessKey.ValidationStatus = Valid
+				secretKey.ValidationStatus = Valid
+			} else {
+				accessKey.ValidationStatus = Revoked
+				secretKey.ValidationStatus = Revoked
+			}
+		}
+	}
+}
+
+func alibabaRequest(accessKey, secretKey string) (int, error) {
+	req, err := http.NewRequest("GET", "https://ecs.aliyuncs.com/", nil)
+	if err != nil {
+		return 0, err
+	}
+
+	// Workaround for gitleaks returns the key ends with "
+	accessKey = strings.TrimSuffix(accessKey, "\"")
+	secretKey = strings.TrimSuffix(secretKey, "\"")
+
+	params := req.URL.Query()
+	params.Add("AccessKeyId", accessKey)
+	params.Add("Action", "DescribeRegions")
+	params.Add("SignatureMethod", "HMAC-SHA1")
+	params.Add("SignatureNonce", strconv.FormatInt(time.Now().UnixNano(), 10))
+	params.Add("SignatureVersion", "1.0")
+	params.Add("Timestamp", time.Now().UTC().Format(time.RFC3339))
+	params.Add("Version", "2014-05-26")
+
+	stringToSign := "GET&%2F&" + url.QueryEscape(params.Encode())
+	hmac := hmac.New(sha1.New, []byte(secretKey+"&"))
+	hmac.Write([]byte(stringToSign))
+	signature := base64.StdEncoding.EncodeToString(hmac.Sum(nil))
+
+	params.Add("Signature", signature)
+	req.URL.RawQuery = params.Encode()
+
+	client := &http.Client{}
+	resp, err := client.Do(req)
+	if err != nil {
+		return 0, err
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		defer resp.Body.Close()
+		body, _ := io.ReadAll(resp.Body)
+		log.Panic().
+			Str("access_key", accessKey).
+			Str("service", "alibaba").
+			Int("status_code", resp.StatusCode).
+			Msgf("Failed to validate secret %s", body)
+	}
+
+	log.Debug().Str("service", "alibaba").Int("status_code", resp.StatusCode).Msg("Validated secret")
+	return resp.StatusCode, nil
+}

secrets/engine.go

@@ -19,21 +19,22 @@ import (
 )
 
 type Engine struct {
-	rules    map[string]config.Rule
-	detector detect.Detector
+	rules     map[string]config.Rule
+	detector  detect.Detector
+	validator Validator
 }
 
 const customRegexRuleIdFormat = "custom-regex-%d"
 
-type SecretsConfig struct {
+type EngineConfig struct {
 	SelectedList []string
 	IgnoreList   []string
 	SpecialList  []string
 
 	MaxTargetMegabytes int
 }
 
-func Init(secretsConfig SecretsConfig) (*Engine, error) {
+func Init(secretsConfig EngineConfig) (*Engine, error) {
 	selectedRules := rules.FilterRules(secretsConfig.SelectedList, secretsConfig.IgnoreList, secretsConfig.SpecialList)
 	if len(*selectedRules) == 0 {
 		return nil, fmt.Errorf("no rules were selected")
@@ -52,8 +53,9 @@ func Init(secretsConfig SecretsConfig) (*Engine, error) {
 	detector.MaxTargetMegaBytes = secretsConfig.MaxTargetMegabytes
 
 	return &Engine{
-		rules:    rulesToBeApplied,
-		detector: *detector,
+		rules:     rulesToBeApplied,
+		detector:  *detector,
+		validator: *NewValidator(),
 	}, nil
 }
 
@@ -100,6 +102,15 @@ func (s *Engine) AddRegexRules(patterns []string) error {
 	return nil
 }
 
+func (s *Engine) RegisterForValidation(secret *Secret, wg *sync.WaitGroup) {
+	defer wg.Done()
+	s.validator.RegisterForValidation(secret)
+}
+
+func (s *Engine) Validate() {
+	s.validator.Validate()
+}
+
 func getFindingId(item plugins.Item, finding report.Finding) string {
 	idParts := []string{item.ID, finding.RuleID, finding.Secret}
 	sha := sha1.Sum([]byte(strings.Join(idParts, "-")))
@@ -115,7 +126,7 @@ func isSecretIgnored(secret *Secret, ignoredIds *[]string) bool {
 	return false
 }
 
-func GetRulesCommand(secretsConfig *SecretsConfig) *cobra.Command {
+func GetRulesCommand(secretsConfig *EngineConfig) *cobra.Command {
 	canValidateDisplay := map[bool]string{
 		true:  "V",
 		false: "",

secrets/engine_test.go

@@ -15,12 +15,12 @@ func Test_Init(t *testing.T) {
 
 	tests := []struct {
 		name          string
-		secretsConfig SecretsConfig
+		secretsConfig EngineConfig
 		expectedErr   error
 	}{
 		{
 			name: "selected and ignore flags used together for the same rule",
-			secretsConfig: SecretsConfig{
+			secretsConfig: EngineConfig{
 				SelectedList: []string{allRules[0].Rule.RuleID},
 				IgnoreList:   []string{allRules[0].Rule.RuleID},
 				SpecialList:  []string{},
@@ -29,7 +29,7 @@ func Test_Init(t *testing.T) {
 		},
 		{
 			name: "non existent select flag",
-			secretsConfig: SecretsConfig{
+			secretsConfig: EngineConfig{
 				SelectedList: []string{"non-existent-tag-name"},
 				IgnoreList:   []string{},
 				SpecialList:  []string{"non-existent-tag-name"},
@@ -38,7 +38,7 @@ func Test_Init(t *testing.T) {
 		},
 		{
 			name: "exiting special rule",
-			secretsConfig: SecretsConfig{
+			secretsConfig: EngineConfig{
 				SelectedList: []string{"non-existent-tag-name"},
 				IgnoreList:   []string{},
 				SpecialList:  []string{specialRule.RuleID},
@@ -110,7 +110,7 @@ func TestSecrets(t *testing.T) {
 		},
 	}
 
-	detector, err := Init(SecretsConfig{})
+	detector, err := Init(EngineConfig{})
 	if err != nil {
 		t.Fatal(err)
 	}

secrets/pairs.go

@@ -0,0 +1,64 @@
+package secrets
+
+import (
+	"sync"
+)
+
+type pairsByRuleId map[string][]*Secret
+type pairsBySource map[string]pairsByRuleId
+type pairsByGeneralKey map[string]pairsBySource
+
+type pairsCollector struct {
+	pairs pairsByGeneralKey
+}
+
+func newPairsCollector() *pairsCollector {
+	return &pairsCollector{pairs: make(pairsByGeneralKey)}
+}
+
+func (p *pairsCollector) addIfNeeded(secret *Secret) bool {
+	generalKey, ok := ruleToGeneralKey[secret.RuleID]
+	if !ok {
+		return false
+	}
+
+	if _, ok := p.pairs[generalKey]; !ok {
+		p.pairs[generalKey] = make(pairsBySource)
+	}
+	if _, ok := p.pairs[generalKey][secret.Source]; !ok {
+		p.pairs[generalKey][secret.Source] = make(pairsByRuleId)
+	}
+	if _, ok := p.pairs[generalKey][secret.Source][secret.RuleID]; !ok {
+		p.pairs[generalKey][secret.Source][secret.RuleID] = make([]*Secret, 0)
+	}
+
+	p.pairs[generalKey][secret.Source][secret.RuleID] = append(p.pairs[generalKey][secret.Source][secret.RuleID], secret)
+	return true
+}
+
+func (p *pairsCollector) validate(generalKey string, rulesById pairsByRuleId, wg *sync.WaitGroup) {
+	defer wg.Done()
+	generalKeyToValidation[generalKey](rulesById)
+}
+
+type pairsValidationFunc func(pairsByRuleId)
+
+var generalKeyToValidation = map[string]pairsValidationFunc{
+	"alibaba": validateAlibaba,
+}
+
+var generalKeyToRules = map[string][]string{
+	"alibaba": {"alibaba-access-key-id", "alibaba-secret-key"},
+}
+
+func generateRuleToGeneralKey() map[string]string {
+	ruleToGeneralKey := make(map[string]string)
+	for key, rules := range generalKeyToRules {
+		for _, rule := range rules {
+			ruleToGeneralKey[rule] = key
+		}
+	}
+	return ruleToGeneralKey
+}
+
+var ruleToGeneralKey = generateRuleToGeneralKey()

secrets/secret.go

@@ -1,19 +1,7 @@
 package secrets
 
 import (
-	"fmt"
-	"net/http"
 	"sync"
-
-	"github.com/rs/zerolog/log"
-)
-
-type ValidationResult string
-
-const (
-	Valid   ValidationResult = "Valid"
-	Revoked ValidationResult = "Revoked"
-	Unknown ValidationResult = "Unknown"
 )
 
 type Secret struct {
@@ -28,13 +16,6 @@ type Secret struct {
 	ValidationStatus ValidationResult `json:"validationStatus,omitempty"`
 }
 
-type validationFunc = func(*Secret) ValidationResult
-
-var ruleIDToFunction = map[string]validationFunc{
-	"github-fine-grained-pat": validateGithub,
-	"github-pat":              validateGithub,
-}
-
 func isCanValidateRule(ruleID string) bool {
 	_, ok := ruleIDToFunction[ruleID]
 	return ok
@@ -48,26 +29,3 @@ func (s *Secret) Validate(wg *sync.WaitGroup) {
 		s.ValidationStatus = Unknown
 	}
 }
-
-func validateGithub(s *Secret) ValidationResult {
-	const githubURL = "https://api.github.com/"
-
-	req, err := http.NewRequest("GET", githubURL, nil)
-	if err != nil {
-		log.Warn().Err(err).Msg("Failed to validate secret")
-		return Unknown
-	}
-	req.Header.Set("Authorization", fmt.Sprintf("token %s", s.Value))
-
-	client := &http.Client{}
-	resp, err := client.Do(req)
-	if err != nil {
-		log.Warn().Err(err).Msg("Failed to validate secret")
-		return Unknown
-	}
-
-	if resp.StatusCode == http.StatusOK {
-		return Valid
-	}
-	return Revoked
-}