Merge branch 'master' into cloudflare

.github/workflows/test.yml

@@ -1,9 +1,6 @@
 name: Test
 
 on:
-  push:
-    branches:
-      - "*"
   pull_request:
     branches:
       - "*"

cmd/generate/config/main.go

@@ -36,7 +36,8 @@ func main() {
 		rules.AsanaClientSecret(),
 		rules.Atlassian(),
 		rules.Authress(),
-		rules.AWS(),
+		rules.AWSAccessKey(),
+		rules.AWSSecretKey(),
 		rules.BitBucketClientID(),
 		rules.BitBucketClientSecret(),
 		rules.BittrexAccessKey(),
@@ -70,7 +71,9 @@ func main() {
 		rules.EasyPost(),
 		rules.EasyPostTestAPI(),
 		rules.EtsyAccessToken(),
-		rules.Facebook(),
+		rules.FacebookSecret(),
+		rules.FacebookAccessToken(),
+		rules.FacebookPageAccessToken(),
 		rules.FastlyAPIToken(),
 		rules.FinicityClientSecret(),
 		rules.FinicityAPIToken(),

cmd/generate/config/rules/adobe.go

@@ -28,6 +28,7 @@ func AdobeClientSecret() *config.Rule {
 		RuleID:      "adobe-client-secret",
 		Regex:       generateUniqueTokenRegex(`(p8e-)(?i)[a-z0-9]{32}`, true),
 		Keywords:    []string{"p8e-"},
+		SecretGroup: 1,
 	}
 
 	// validate

cmd/generate/config/rules/alibaba.go

@@ -12,6 +12,7 @@ func AlibabaAccessKey() *config.Rule {
 		RuleID:      "alibaba-access-key-id",
 		Regex:       generateUniqueTokenRegex(`(LTAI)(?i)[a-z0-9]{20}`, true),
 		Keywords:    []string{"LTAI"},
+		SecretGroup: 1,
 	}
 
 	// validate

cmd/generate/config/rules/aws.go

@@ -1,18 +1,22 @@
 package rules
 
 import (
-	"regexp"
-
+	"github.com/zricethezav/gitleaks/v8/cmd/generate/secrets"
 	"github.com/zricethezav/gitleaks/v8/config"
 )
 
-func AWS() *config.Rule {
+// https://summitroute.com/blog/2018/06/20/aws_security_credential_formats/
+
+var credFileAccessKey = "aws_access_key_id=AKIALALEMEL33243OLIB" // gitleaks:allow
+var credFileSecretKey = "aws_secret_access_key=" + secrets.NewSecret(hex("40"))
+var credFileSessionToken = "aws_session_token=" + secrets.NewSecret(hex("928"))
+
+func AWSAccessKey() *config.Rule {
 	// define rule
 	r := config.Rule{
 		Description: "Identified a pattern that may indicate AWS credentials, risking unauthorized cloud resource access and data breaches on AWS platforms.",
-		RuleID:      "aws-access-token",
-		Regex: regexp.MustCompile(
-			"(?:A3T[A-Z0-9]|AKIA|ASIA|ABIA|ACCA)[A-Z0-9]{16}"),
+		RuleID:      "aws-access-key",
+		Regex:       generateUniqueTokenRegex("(?:A3T[A-Z0-9]|AKIA|ASIA|ABIA|ACCA)[A-Z2-7]{16}", false),
 		Keywords: []string{
 			"AKIA",
 			"ASIA",
@@ -22,6 +26,34 @@ func AWS() *config.Rule {
 	}
 
 	// validate
-	tps := []string{generateSampleSecret("AWS", "AKIALALEMEL33243OLIB")} // gitleaks:allow
-	return validate(r, tps, nil)
+	tps := []string{
+		generateSampleSecret("AWS", "AKIALALEMEL33243OLIB"), // gitleaks:allow
+		credFileAccessKey,
+	}
+	fps := []string{
+		generateSampleSecret("AWS", "AKIALALEMEL33243O000"), // includes 0 which can't be result of base32 encoding
+		`"RoleId": "AROAWORVRXQ5NC76T7223"`,
+		credFileSecretKey,
+		credFileSessionToken,
+	}
+	return validate(r, tps, fps)
+}
+
+func AWSSecretKey() *config.Rule {
+	// define rule
+	r := config.Rule{
+		Description: "Identified a pattern that may indicate AWS credentials, risking unauthorized cloud resource access and data breaches on AWS platforms.",
+		RuleID:      "aws-secret-key",
+		Regex:       generateUniqueTokenRegex("[0-9A-Z+\\/]{40}", true),
+	}
+
+	// validate
+	tps := []string{
+		credFileSecretKey,
+	}
+	fps := []string{
+		credFileAccessKey,
+		credFileSessionToken,
+	}
+	return validate(r, tps, fps)
 }

cmd/generate/config/rules/facebook.go

@@ -5,11 +5,13 @@ import (
 	"github.com/zricethezav/gitleaks/v8/config"
 )
 
-func Facebook() *config.Rule {
+// This rule includes both App Secret and Client Access Token
+// https://developers.facebook.com/docs/facebook-login/guides/access-tokens/
+func FacebookSecret() *config.Rule {
 	// define rule
 	r := config.Rule{
-		Description: "Discovered a Facebook Access Token, posing a risk of unauthorized access to Facebook accounts and personal data exposure.",
-		RuleID:      "facebook",
+		Description: "Discovered a Facebook Application secret, posing a risk of unauthorized access to Facebook accounts and personal data exposure.",
+		RuleID:      "facebook-secret",
 		Regex:       generateSemiGenericRegex([]string{"facebook"}, hex("32"), true),
 
 		Keywords: []string{"facebook"},
@@ -18,6 +20,46 @@ func Facebook() *config.Rule {
 	// validate
 	tps := []string{
 		generateSampleSecret("facebook", secrets.NewSecret(hex("32"))),
+		`facebook_app_secret = "6dca6432e45d933e13650d1882bd5e69"`,       // gitleaks:allow
+		`facebook_client_access_token: 26f5fd13099f2c1331aafb86f6489692`, // gitleaks:allow
+	}
+	return validate(r, tps, nil)
+}
+
+// https://developers.facebook.com/docs/facebook-login/guides/access-tokens/#apptokens
+func FacebookAccessToken() *config.Rule {
+	// define rule
+	r := config.Rule{
+		Description: "Discovered a Facebook Access Token, posing a risk of unauthorized access to Facebook accounts and personal data exposure.",
+		RuleID:      "facebook-access-token",
+		Regex:       generateUniqueTokenRegex(`\d{15,16}\|[0-9a-z\-_]{27}`, true),
+	}
+
+	// validate
+	tps := []string{
+		`{"access_token":"911602140448729|AY-lRJZq9BoDLobvAiP25L7RcMg","token_type":"bearer"}`, // gitleaks:allow
+		`1308742762612587|rhoK1cbv0DOU_RTX_87O4MkX7AI`,                                         // gitleaks:allow
+		`1477036645700765|wRPf2v3mt2JfMqCLK8n7oltrEmc`,                                         // gitleaks:allow
+	}
+	return validate(r, tps, nil)
+}
+
+// https://developers.facebook.com/docs/facebook-login/guides/access-tokens/#pagetokens
+func FacebookPageAccessToken() *config.Rule {
+	// define rule
+	r := config.Rule{
+		Description: "Discovered a Facebook Page Access Token, posing a risk of unauthorized access to Facebook accounts and personal data exposure.",
+		RuleID:      "facebook-page-access-token",
+		Regex:       generateUniqueTokenRegex("EAA[MC]"+alphaNumeric("20,"), true),
+		Keywords:    []string{"EAAM", "EAAC"},
+	}
+
+	// validate
+	tps := []string{
+		`EAAM9GOnCB9kBO2frzOAWGN2zMnZClQshlWydZCrBNdodesbwimx1mfVJgqZBP5RSpMfUzWhtjTTXHG5I1UlvlwRZCgjm3ZBVGeTYiqAAoxyED6HaUdhpGVNoPUwAuAWWFsi9OvyYBQt22DGLqMIgD7VktuCTTZCWKasz81Q822FPhMTB9VFFyClNzQ0NLZClt9zxpsMMrUZCo1VU1rL3CKavir5QTfBjfCEzHNlWAUDUV2YZD`, // gitleaks:allow
+		`EAAM9GOnCB9kBO2zXpAtRBmCrsPPjdA3KeBl4tqsEpcYd09cpjm9MZCBIklZBjIQBKGIJgFwm8IE17G5pipsfRBRBEHMWxvJsL7iHLUouiprxKRQfAagw8BEEDucceqxTiDhVW2IZAQNNbf0d1JhcapAGntx5S1Csm4j0GgZB3DuUfI2HJ9aViTtdfH2vjBy0wtpXm2iamevohGfoF4NgyRHusDLjqy91uYMkfrkc`,          // gitleaks:allow
+		`- name: FACEBOOK_TOKEN
+		value: "EAACEdEose0cBA1bad3afsf2aew"`, // gitleaks:allow
 	}
 	return validate(r, tps, nil)
 }

cmd/generate/config/rules/generic.go

@@ -43,12 +43,28 @@ func GenericCredential() *config.Rule {
 		generateSampleSecret("generic", "Zf3D0LXCM3EIMbgJpUNnkRtOfOueHznB"),
 		`"client_id" : "0afae57f3ccfd9d7f5767067bc48b30f719e271ba470488056e37ab35d4b6506"`,
 		`"client_secret" : "6da89121079f83b2eb6acccf8219ea982c3d79bccc3e9c6a85856480661f8fde",`,
+		`"password: 'edf8f16608465858a6c9e3cccb97d3c2'"`,
+		"<password>edf8f16608465858a6c9e3cccb97d3c2</password>",
+		`<element password="edf8f16608465858a6c9e3cccb97d3c2" />`,
+		"M_DB_PASSWORD= edf8f16608465858a6c9e3cccb97d3c2",
+		`{ "access-key": "6da89121079f83b2eb6acccf8219ea982c3d79bccc", }`,
+		`"{ \"access-key\": \"6da89121079f83b2eb6acccf8219ea982c3d79bccc\", }"`,
 	}
 	fps := []string{
 		`client_vpn_endpoint_id = aws_ec2_client_vpn_endpoint.client-vpn-endpoint.id`,
 		`password combination.
 
-R5: Regulatory--21`,
+		R5: Regulatory--21`,
+
+		`"client_id" : "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"`,
+		`"client_secret" : "4v7b9n2k5h",`, // entropy: 3.32
+		`"password: 'comp123!'"`,
+		"<password>MyComp9876</password>", // entropy: 3.32
+		`<element password="Comp4567@@" />`,
+		"M_DB_PASSWORD= aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+		"GITHUB_TOKEN: ${GITHUB_TOKEN}",
+		"password = 'your_password_here'",
+		"https://google.com?user=abc&password=123",
 	}
 	return validate(r, tps, fps)
 }

cmd/generate/config/rules/hashicorp.go

@@ -30,8 +30,9 @@ func HashicorpField() *config.Rule {
 	r := config.Rule{
 		Description: "Identified a HashiCorp Terraform password field, risking unauthorized infrastructure configuration and security breaches.",
 		RuleID:      "hashicorp-tf-password",
-		Regex:       generateSemiGenericRegex(keywords, fmt.Sprintf(`"%s"`, alphaNumericExtended("8,20")), true),
+		Regex:       generateSemiGenericRegex(keywords, fmt.Sprintf(`"(%s)"`, alphaNumericExtended("8,20")), true),
 		Keywords:    keywords,
+		SecretGroup: 2,
 	}
 
 	tps := []string{

cmd/generate/config/rules/lob.go

@@ -17,6 +17,7 @@ func LobPubAPIToken() *config.Rule {
 			"live_pub",
 			"_pub",
 		},
+		SecretGroup: 1,
 	}
 
 	// validate
@@ -36,6 +37,7 @@ func LobAPIToken() *config.Rule {
 			"test_",
 			"live_",
 		},
+		SecretGroup: 1,
 	}
 
 	// validate

cmd/generate/config/rules/mailchimp.go

@@ -10,7 +10,7 @@ func MailChimp() *config.Rule {
 	r := config.Rule{
 		RuleID:      "mailchimp-api-key",
 		Description: "Identified a Mailchimp API key, potentially compromising email marketing campaigns and subscriber data.",
-		Regex:       generateSemiGenericRegex([]string{"mailchimp"}, `[a-f0-9]{32}-us20`, true),
+		Regex:       generateSemiGenericRegex([]string{"MailchimpSDK.initialize", "mailchimp"}, hex("32")+`-us\d\d`, true),
 
 		Keywords: []string{
 			"mailchimp",
@@ -20,6 +20,12 @@ func MailChimp() *config.Rule {
 	// validate
 	tps := []string{
 		generateSampleSecret("mailchimp", secrets.NewSecret(hex("32"))+"-us20"),
+		`mailchimp_api_key: cefa780880ba5f5696192a34f6292c35-us18`, // gitleaks:allow
+		`MAILCHIMPE_KEY = "b5b9f8e50c640da28993e8b6a48e3e53-us18"`, // gitleaks:allow
 	}
-	return validate(r, tps, nil)
+	fps := []string{
+		// False Negative
+		`MailchimpSDK.initialize(token: 3012a5754bbd716926f99c028f7ea428-us18)`, // gitleaks:allow
+	}
+	return validate(r, tps, fps)
 }

cmd/generate/config/rules/rule.go

@@ -18,16 +18,16 @@ const (
 	identifierCaseInsensitivePrefix = `(?i:`
 	identifierCaseInsensitiveSuffix = `)`
 	identifierPrefix                = `(?:`
-	identifierSuffix                = `)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}`
+	identifierSuffix                = `)(?:[0-9a-z\-_\t .]{0,20})(?:[\s'"\\]){0,3}`
 
 	// commonly used assignment operators or function call
 	operator = `(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)`
 
 	// boundaries for the secret
 	// \x60 = `
 	secretPrefixUnique = `\b(`
-	secretPrefix       = `(?:'|\"|\s|=|\x60){0,5}(`
-	secretSuffix       = `)(?:['|\"|\n|\r|\s|\x60|;]|$)`
+	secretPrefix       = `(?:['\"\\\s=\x60]){0,5}(`
+	secretSuffix       = `)(?:['\"\\\n\r\s\x60;<]|$)`
 )
 
 func generateSemiGenericRegex(identifiers []string, secretRegex string, isCaseInsensitive bool) *regexp.Regexp {

cmd/generate/config/rules/scalingo.go

@@ -1,8 +1,6 @@
 package rules
 
 import (
-	"regexp"
-
 	"github.com/zricethezav/gitleaks/v8/cmd/generate/secrets"
 	"github.com/zricethezav/gitleaks/v8/config"
 )
@@ -12,13 +10,14 @@ func ScalingoAPIToken() *config.Rule {
 	r := config.Rule{
 		Description: "Found a Scalingo API token, posing a risk to cloud platform services and application deployment security.",
 		RuleID:      "scalingo-api-token",
-		Regex:       regexp.MustCompile(`\btk-us-[a-zA-Z0-9-_]{48}\b`),
+		Regex:       generateUniqueTokenRegex(`tk-us-[a-zA-Z0-9-_]{48}`, false),
 		Keywords:    []string{"tk-us-"},
 	}
 
 	// validate
 	tps := []string{
 		generateSampleSecret("scalingo", "tk-us-"+secrets.NewSecret(alphaNumericExtendedShort("48"))),
+		`scalingo_api_token = "tk-us-loys7ib9yrxcys_ta2sq85mjar6lgcsspkd9x61s7h5epf_-"`, // gitleaks:allow
 	}
 	return validate(r, tps, nil)
 }

cmd/generate/config/rules/shippo.go

@@ -15,6 +15,7 @@ func ShippoAPIToken() *config.Rule {
 		Keywords: []string{
 			"shippo_",
 		},
+		SecretGroup: 1,
 	}
 
 	// validate