From 0e4c7963fa5f93e13e76552ec02d043323ffc658 Mon Sep 17 00:00:00 2001
From: aegilops <41705651+aegilops@users.noreply.github.com>
Date: Thu, 18 Apr 2024 14:00:23 +0100
Subject: [PATCH 01/95] Precise Okta token and Google private key id or old API
 key

---
 vendors/patterns.yml | 36 +++++++++++++++++++++++++++++++++++-
 1 file changed, 35 insertions(+), 1 deletion(-)

diff --git a/vendors/patterns.yml b/vendors/patterns.yml
index 0902683f..8b59fee7 100644
--- a/vendors/patterns.yml
+++ b/vendors/patterns.yml
@@ -227,10 +227,26 @@ patterns:
       - name: okta.txt
         start_offset: 180
         end_offset: 222
-
     comments:
       - Okta token, starting with `00` and 40 random alphanumeric with _ and -
 
+  - name: Okta API key (precise)
+    type: okta_token_precise
+    regex:
+      version: 0.1
+      pattern: |
+        0{2}[0-9A-Za-z_-]{40}
+      start: |
+        (\bSSWS\s{1,5}|(?i)okta[_-]?(api[_-]?)?(token|key|secret)\s{0,32}([:=]|[=-]>|to|[!=]={1,2}|<>)\s{0,32}['"`]?)
+      end: |
+        \z|[^0-9A-Za-z_+/=-]
+    test:
+      data: SSWS 00RCsRGyrF9QzBe0rnWmw221UeclqRXFQGEJFdhAFM
+      start_offset: 5
+    comments:
+      - Uses surrounding context to reduce false positives
+      - Either `SSWS ` then the token, or a variable starting `okta` followed by an assignment operator, then the token
+
   - name: DataDog API key
     type: datadog_api_key
     regex:
@@ -534,3 +550,21 @@ patterns:
       data: ClientSecret = "kc3KP75a~edN1cV.rs~_r6F-O7-C-55-Ts"
       start_offset: 16
       end_offset: 50
+
+  - name: Google private key id (or older API key)
+    type: google_private_key_id_or_old_api_token
+    regex:
+      version: 0.1
+      pattern: |
+        [a-fA-F0-9]{40}
+      start: |
+        (?i)(private_key_id|google_api_key)['"`]?(\s*[\]\)])?\s*([:,=]|[=-]>|to|[!=]={1,2}|<>)?\s*([[{])?['"`]?
+      end: |
+        \b|\z
+    test:
+      data: |
+        google_api_key: 0123456789abcdef0123456789abcdef01234567
+      start_offset: 16
+      end_offset: -1
+
+  - 
\ No newline at end of file

From 24b3f223a044e4bc1487705cfcd63ad0b2c895ab Mon Sep 17 00:00:00 2001
From: aegilops <41705651+aegilops@users.noreply.github.com>
Date: Thu, 18 Apr 2024 14:01:12 +0100
Subject: [PATCH 02/95] Norwegian national ID number

---
 pii/patterns.yml | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/pii/patterns.yml b/pii/patterns.yml
index a31642f0..7b8f7ad3 100644
--- a/pii/patterns.yml
+++ b/pii/patterns.yml
@@ -237,3 +237,21 @@ patterns:
       - Examples include YouTube playlist IDs and Italian tax codes
       - If this is a problem, try the individual country IBAN patterns, based on the published IBAN structures on https://iban.com/structure
 
+  - name: Norwegian national identity number/D number
+    type: no_national_id_number
+    regex:
+      pattern: |
+        (([04][1-9]|[15][0-9]|[26][0-9])(0[1-9]|1[0-2])|[37]0(0[469]|11)|[37][01](0[13578]|1[02]))[0-9]{2} ?[0-9]{3} ?[0-9]{2}
+      start: |
+        \A|[^0-9A-Za-z_.+/\\-]
+      end: |
+        \z|[^0-9A-Za-z_.+/\\=-]
+      additional_not_match:
+        - 1111111111[123]|11112222333|01123456978|410185 ?123 ?45|220676 ?123 ?45|01010202010|01010101023
+    test:
+      data: |
+        41018512346
+    comments:
+      - With no validation of the checksum this can cause a lot of false positives
+      - The example test data does not have a valid checksum - it is one of the examples used with one digit in the checksum changed
+      - You can test using the correct checksum, but it is used as a NOT match here to prevent false positives on other test data

From 7b8cf9d2e460ea856557db4eadd7b987646a9103 Mon Sep 17 00:00:00 2001
From: aegilops <41705651+aegilops@users.noreply.github.com>
Date: Thu, 18 Apr 2024 17:17:08 +0100
Subject: [PATCH 03/95] OpenStack password or API key

---
 vendors/patterns.yml | 22 +++++++++++++++++++++-
 1 file changed, 21 insertions(+), 1 deletion(-)

diff --git a/vendors/patterns.yml b/vendors/patterns.yml
index 8b59fee7..5ac99600 100644
--- a/vendors/patterns.yml
+++ b/vendors/patterns.yml
@@ -567,4 +567,24 @@ patterns:
       start_offset: 16
       end_offset: -1
 
-  - 
\ No newline at end of file
+  - name: OpenStack password/API key
+    type: openstack_password_or_key
+    description: OpenStack password or API key
+    regex:
+      pattern: |
+        [^'",\r\n\x00-\x08]+
+      start: |
+        (?i)OPEN_?STACK_(PASSWORD|API_?KEY)[_A-Z]*['"`]?(\s*[\]\)])?\s*([:,=]|[=-]>|to|[!=]={1,2}|<>)?\s*([[{])?['"`]?
+      end: |
+        ['"\r\n,]|\z
+      additional_not_match:
+        - ^(ENV|[a-z_]+)\[$
+        - ^<%=.*%>$
+        - ^([a-z_]+\.api_?key|self\.[a-z_]+|os\.environ\.get\()$
+        - ^(\$\{?[A-Z]+\}?|<password>)$
+        - ^(@?[a-z_]+\[:.*\]|@[a-z_]+)$
+    test:
+      data: |
+        OPENSTACK_PASSWORD: "password1234isnotagoodpassword"
+      start_offset: 21
+      end_offset: 51

From aba524b57976cf251c9cd0c178a001d6fdda077c Mon Sep 17 00:00:00 2001
From: aegilops <41705651+aegilops@users.noreply.github.com>
Date: Thu, 18 Apr 2024 18:34:52 +0100
Subject: [PATCH 04/95] Database connection strings

---
 database/patterns.yml | 35 ++++++++++++++++++++++++++++++++++-
 1 file changed, 34 insertions(+), 1 deletion(-)

diff --git a/database/patterns.yml b/database/patterns.yml
index a67da06a..4f9d09f0 100644
--- a/database/patterns.yml
+++ b/database/patterns.yml
@@ -113,5 +113,38 @@ patterns:
     comments:
       - This is specific to Microsoft SQL Server TSQL syntax
 
+  - name: SQLAlchemy Database Connection String
+    type: sqlalchemy_connection_string
+    description: SQLAlchemy connection strings are used to connect to databases, often with embedded credentials.
+    regex:
+      pattern: |
+        [^$/?#@\s][^/?#@\s\x00-\x08]*
+      start: |
+        (\A|\b)mysql\+[a-z]+://[^/?#:@\s\x00-\x08]*:
+      end: |
+        @
+      additional_not_match:
+        - (?i)^[[{(<]?(?:password|passwd|secret)[\]})>]?$
+        - ^\$?\{[^}+]\}i\}$
+        - ^%(?:\.\*)?s$
+    test:
+      data: |
+        engine = create_engine('mysql+mysqlconnector://scott:tiger@localhost/foo')
+      start_offset: 53
+      end_offset: 58
 
-
+  - name: MongoDB Database Connection String
+    type: mongodb_connection_string
+    description: MongoDB connection strings are used to connect to databases, often with embedded credentials.
+    regex:
+      pattern: |
+        [^'"<>/@\s\x00-\x08]+
+      start: |
+        (\A|\b)mongodb(\+[a-z]+)?://[^'"<>/:@\s\x00-\x08]+:
+      end: |
+        @
+      additional_not_match:
+        - ^((test-?|a|my)?pass(word)?|%244to%40L8%3DMC)$
+        - ^%(?:\.\*)?[sv]$
+        - ^\$?\{[^}+]\}$
+        - ^\$[A-Za-z_]+$

From 5e453d5d93e498025398762c891bbc290d75d185 Mon Sep 17 00:00:00 2001
From: aegilops <41705651+aegilops@users.noreply.github.com>
Date: Thu, 18 Apr 2024 18:39:45 +0100
Subject: [PATCH 05/95] Updated LaunchDarkly to include SDK keys

---
 vendors/patterns.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/vendors/patterns.yml b/vendors/patterns.yml
index 5ac99600..4f688181 100644
--- a/vendors/patterns.yml
+++ b/vendors/patterns.yml
@@ -344,10 +344,11 @@ patterns:
 
   - name: LaunchDarkly API key
     type: launchdarkly_api_key
+    description: LaunchDarkly API or SDK key
     regex:
       version: 0.1
       pattern: |
-        api-[a-f0-9-]{8}-[a-f0-9-]{4}-[a-f0-9-]{4}-[a-f0-9-]{4}-[a-f0-9-]{12}
+        (api|sdk)-[a-f0-9-]{8}-[a-f0-9-]{4}-[a-f0-9-]{4}-[a-f0-9-]{4}-[a-f0-9-]{12}
       start: |
         \A|\b
       end: |

From af275507106d510b11ba4f752de2ff32bbb3115b Mon Sep 17 00:00:00 2001
From: aegilops <41705651+aegilops@users.noreply.github.com>
Date: Fri, 19 Apr 2024 10:57:04 +0100
Subject: [PATCH 06/95] Updated README.md

---
 README.md          |  12 ++++
 database/README.md | 119 +++++++++++++++++++++++++++++++++++++
 pii/README.md      |  55 +++++++++++++++++
 vendors/README.md  | 143 ++++++++++++++++++++++++++++++++++++++++++++-
 4 files changed, 327 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 7abf3a08..52248e2d 100644
--- a/README.md
+++ b/README.md
@@ -41,6 +41,10 @@ Custom Secret Scanning Patterns repository.
 - Database Connection String (3)
 
 - TSQL CREATE LOGIN/USER
+
+- SQLAlchemy Database Connection String
+
+- MongoDB Database Connection String
   
 
 ### [Generic Secrets / Passwords](./generic)
@@ -85,6 +89,8 @@ Custom Secret Scanning Patterns repository.
 - Credit Cards - Discover
 
 - IBAN
+
+- Norwegian national identity number/D number
   
 
 ### [RSA Keys](./rsa)
@@ -135,6 +141,8 @@ Custom Secret Scanning Patterns repository.
 
 - Okta token
 
+- Okta API key (precise)
+
 - DataDog API key
 
 - DataDog APP key
@@ -168,4 +176,8 @@ Custom Secret Scanning Patterns repository.
 - UUIDv4 Bearer token (maybe Heroku)
 
 - Azure client secret
+
+- Google private key id (or older API key)
+
+- OpenStack password/API key
   
\ No newline at end of file
diff --git a/database/README.md b/database/README.md
index c84a0541..ba29833b 100644
--- a/database/README.md
+++ b/database/README.md
@@ -207,4 +207,123 @@ _version: v0.1_
 \'
 ```
 
+</details>
+
+## SQLAlchemy Database Connection String
+
+
+SQLAlchemy connection strings are used to connect to databases, often with embedded credentials.
+_version: v0.1_
+
+
+
+<details>
+<summary>Pattern Format</summary>
+
+```regex
+[^$/?#@\s][^/?#@\s\x00-\x08]*
+```
+
+</details>
+
+<details>
+<summary>Start Pattern</summary>
+
+```regex
+(\A|\b)mysql\+[a-z]+://[^/?#:@\s\x00-\x08]*:
+```
+
+</details><details>
+<summary>End Pattern</summary>
+
+```regex
+@
+```
+
+</details>
+
+<details>
+<summary>Additional Matches</summary>
+
+Add these additional matches to the [Secret Scanning Custom Pattern](https://docs.github.com/en/enterprise-cloud@latest/code-security/secret-scanning/defining-custom-patterns-for-secret-scanning#example-of-a-custom-pattern-specified-using-additional-requirements).
+
+
+- Not Match:
+
+  ```regex
+  (?i)^[[{(<]?(?:password|passwd|secret)[\]})>]?$
+  ```
+- Not Match:
+
+  ```regex
+  ^\$?\{[^}+]\}i\}$
+  ```
+- Not Match:
+
+  ```regex
+  ^%(?:\.\*)?s$
+  ```
+
+</details>
+
+## MongoDB Database Connection String
+
+
+MongoDB connection strings are used to connect to databases, often with embedded credentials.
+_version: v0.1_
+
+
+
+<details>
+<summary>Pattern Format</summary>
+
+```regex
+[^'"<>/@\s\x00-\x08]+
+```
+
+</details>
+
+<details>
+<summary>Start Pattern</summary>
+
+```regex
+(\A|\b)mongodb(\+[a-z]+)?://[^'"<>/:@\s\x00-\x08]+:
+```
+
+</details><details>
+<summary>End Pattern</summary>
+
+```regex
+@
+```
+
+</details>
+
+<details>
+<summary>Additional Matches</summary>
+
+Add these additional matches to the [Secret Scanning Custom Pattern](https://docs.github.com/en/enterprise-cloud@latest/code-security/secret-scanning/defining-custom-patterns-for-secret-scanning#example-of-a-custom-pattern-specified-using-additional-requirements).
+
+
+- Not Match:
+
+  ```regex
+  ^((test-?|a|my)?pass(word)?|%244to%40L8%3DMC)$
+  ```
+- Not Match:
+
+  ```regex
+  ^%(?:\.\*)?[sv]$
+  ```
+- Not Match:
+
+  ```regex
+  ^\$?\{[^}+]\}$
+  ```
+- Not Match:
+
+  ```regex
+  ^\$[A-Za-z_]+$
+  ```
+
 </details>
\ No newline at end of file
diff --git a/pii/README.md b/pii/README.md
index df6363f7..ce0c639c 100644
--- a/pii/README.md
+++ b/pii/README.md
@@ -178,4 +178,59 @@ Add these additional matches to the [Secret Scanning Custom Pattern](https://doc
   ^(?:MG4600005030071289421016045|ML13ML0160120102600100668497|MZ59000301080016367102371|NE58NE0380100100130305000268|SN08SN0100152000048500003035|TD8960002000010271091600153|TG53TG0090604310346500400070|KM4600005000010010904400137|HN54PISA00000000000000123124|NI92BAMC000000000000000003123123|MN580050099123456789)$
   ```
 
+</details>
+
+## Norwegian national identity number/D number
+
+
+
+_version: v0.1_
+
+**Comments / Notes:**
+
+
+- With no validation of the checksum this can cause a lot of false positives
+
+- The example test data does not have a valid checksum - it is one of the examples used with one digit in the checksum changed
+
+- You can test using the correct checksum, but it is used as a NOT match here to prevent false positives on other test data
+  
+
+<details>
+<summary>Pattern Format</summary>
+
+```regex
+(([04][1-9]|[15][0-9]|[26][0-9])(0[1-9]|1[0-2])|[37]0(0[469]|11)|[37][01](0[13578]|1[02]))[0-9]{2} ?[0-9]{3} ?[0-9]{2}
+```
+
+</details>
+
+<details>
+<summary>Start Pattern</summary>
+
+```regex
+\A|[^0-9A-Za-z_.+/\\-]
+```
+
+</details><details>
+<summary>End Pattern</summary>
+
+```regex
+\z|[^0-9A-Za-z_.+/\\=-]
+```
+
+</details>
+
+<details>
+<summary>Additional Matches</summary>
+
+Add these additional matches to the [Secret Scanning Custom Pattern](https://docs.github.com/en/enterprise-cloud@latest/code-security/secret-scanning/defining-custom-patterns-for-secret-scanning#example-of-a-custom-pattern-specified-using-additional-requirements).
+
+
+- Not Match:
+
+  ```regex
+  1111111111[123]|11112222333|01123456978|410185 ?123 ?45|220676 ?123 ?45|01010202010|01010101023
+  ```
+
 </details>
\ No newline at end of file
diff --git a/vendors/README.md b/vendors/README.md
index b44c3c63..73a68d3d 100644
--- a/vendors/README.md
+++ b/vendors/README.md
@@ -387,6 +387,45 @@ Add these additional matches to the [Secret Scanning Custom Pattern](https://doc
 
 </details>
 
+## Okta API key (precise)
+
+
+
+_version: v0.1_
+
+**Comments / Notes:**
+
+
+- Uses surrounding context to reduce false positives
+
+- Either `SSWS ` then the token, or a variable starting `okta` followed by an assignment operator, then the token
+  
+
+<details>
+<summary>Pattern Format</summary>
+
+```regex
+0{2}[0-9A-Za-z_-]{40}
+```
+
+</details>
+
+<details>
+<summary>Start Pattern</summary>
+
+```regex
+(\bSSWS\s{1,5}|(?i)okta[_-]?(api[_-]?)?(token|key|secret)\s{0,32}([:=]|[=-]>|to|[!=]={1,2}|<>)\s{0,32}['"`]?)
+```
+
+</details><details>
+<summary>End Pattern</summary>
+
+```regex
+\z|[^0-9A-Za-z_+/=-]
+```
+
+</details>
+
 ## DataDog API key
 
 
@@ -564,7 +603,7 @@ Add these additional matches to the [Secret Scanning Custom Pattern](https://doc
 ## LaunchDarkly API key
 
 
-
+LaunchDarkly API or SDK key
 _version: v0.1_
 
 
@@ -573,7 +612,7 @@ _version: v0.1_
 <summary>Pattern Format</summary>
 
 ```regex
-api-[a-f0-9-]{8}-[a-f0-9-]{4}-[a-f0-9-]{4}-[a-f0-9-]{4}-[a-f0-9-]{12}
+(api|sdk)-[a-f0-9-]{8}-[a-f0-9-]{4}-[a-f0-9-]{4}-[a-f0-9-]{4}-[a-f0-9-]{12}
 ```
 
 </details>
@@ -1077,4 +1116,104 @@ Add these additional matches to the [Secret Scanning Custom Pattern](https://doc
   [.~_-][A-Za-z0-9]|[A-Za-z0-9][.~_-]
   ```
 
+</details>
+
+## Google private key id (or older API key)
+
+
+
+_version: v0.1_
+
+
+
+<details>
+<summary>Pattern Format</summary>
+
+```regex
+[a-fA-F0-9]{40}
+```
+
+</details>
+
+<details>
+<summary>Start Pattern</summary>
+
+```regex
+(?i)(private_key_id|google_api_key)['"`]?(\s*[\]\)])?\s*([:,=]|[=-]>|to|[!=]={1,2}|<>)?\s*([[{])?['"`]?
+```
+
+</details><details>
+<summary>End Pattern</summary>
+
+```regex
+\b|\z
+```
+
+</details>
+
+## OpenStack password/API key
+
+
+OpenStack password or API key
+_version: v0.1_
+
+
+
+<details>
+<summary>Pattern Format</summary>
+
+```regex
+[^'",\r\n\x00-\x08]+
+```
+
+</details>
+
+<details>
+<summary>Start Pattern</summary>
+
+```regex
+(?i)OPEN_?STACK_(PASSWORD|API_?KEY)[_A-Z]*['"`]?(\s*[\]\)])?\s*([:,=]|[=-]>|to|[!=]={1,2}|<>)?\s*([[{])?['"`]?
+```
+
+</details><details>
+<summary>End Pattern</summary>
+
+```regex
+['"\r\n,]|\z
+```
+
+</details>
+
+<details>
+<summary>Additional Matches</summary>
+
+Add these additional matches to the [Secret Scanning Custom Pattern](https://docs.github.com/en/enterprise-cloud@latest/code-security/secret-scanning/defining-custom-patterns-for-secret-scanning#example-of-a-custom-pattern-specified-using-additional-requirements).
+
+
+- Not Match:
+
+  ```regex
+  ^(ENV|[a-z_]+)\[$
+  ```
+- Not Match:
+
+  ```regex
+  ^<%=.*%>$
+  ```
+- Not Match:
+
+  ```regex
+  ^([a-z_]+\.api_?key|self\.[a-z_]+|os\.environ\.get\()$
+  ```
+- Not Match:
+
+  ```regex
+  ^(\$\{?[A-Z]+\}?|<password>)$
+  ```
+- Not Match:
+
+  ```regex
+  ^(@?[a-z_]+\[:.*\]|@[a-z_]+)$
+  ```
+
 </details>
\ No newline at end of file

From 1c712e42c7ca95dd3faa8778ef2b08715f6fa16e Mon Sep 17 00:00:00 2001
From: aegilops <41705651+aegilops@users.noreply.github.com>
Date: Fri, 19 Apr 2024 11:29:48 +0100
Subject: [PATCH 07/95] Moved MongoDB to databases from vendors and improved it

---
 database/patterns.yml | 18 ++++++++++++++++++
 vendors/patterns.yml  | 18 ------------------
 2 files changed, 18 insertions(+), 18 deletions(-)

diff --git a/database/patterns.yml b/database/patterns.yml
index 4f9d09f0..accfee82 100644
--- a/database/patterns.yml
+++ b/database/patterns.yml
@@ -148,3 +148,21 @@ patterns:
         - ^%(?:\.\*)?[sv]$
         - ^\$?\{[^}+]\}$
         - ^\$[A-Za-z_]+$
+
+  - name: MongoDB connection string
+    type: mongodb_connection_string
+    regex:
+      version: 0.2
+      pattern: |
+        mongodb(\+[a-z]+)?://[^'"`<>/:@\s\x00-\x08]+:[^'"`<>/@\s\x00-\x08]+@[^?'"`\s\x00-\x08]+
+      start: |
+        \A|\b
+      end: |
+        \z|\s|['"`?]
+      additional_not_match:
+        - (?i):(test|a|my)?[_-]?pass(word)?@
+        - :%(?:\.\*)?[sv]@
+        - :\$?\{[^}+]\}@
+        - ^mongodb\+srv://b\*b%40f3tt%3D:%244to%40L8%3DMC@test3.test.build.10gen.cc/mydb%3F\?replicaSet=repl0
+    test:
+      data: mongodb+srv://foo:bar@mongodb.example.invalid/
\ No newline at end of file
diff --git a/vendors/patterns.yml b/vendors/patterns.yml
index 4f688181..1aca0e57 100644
--- a/vendors/patterns.yml
+++ b/vendors/patterns.yml
@@ -501,24 +501,6 @@ patterns:
       start_offset: 57
       end_offset: 81
 
-  - name: MongoDB connection string
-    type: mongodb_connection_string
-    regex:
-      version: 0.1
-      pattern: |
-        mongodb\+srv://[^'"<>/:@\s\x00-\x08]+:[^'"<>/@\s\x00-\x08]+@[^/\s\x00-\x08]+\S*
-      start: |
-        \A|\b
-      end: |
-        \z|\s|['"`]
-      additional_not_match:
-        - ^mongodb\+srv://(test-)?user:(test-)?pass(word)?@
-        - ^mongodb\+srv://%s:%s@
-        - ^mongodb\+srv://auser:apass@
-        - ^mongodb\+srv://b\*b%40f3tt%3D:%244to%40L8%3DMC@test3.test.build.10gen.cc/mydb%3F\?replicaSet=repl0
-    test:
-      data: mongodb+srv://foo:bar@mongodb.example.invalid/
-
   - name: UUIDv4 Bearer token (maybe Heroku)
     type: uuidv4_bearer_token
     regex:

From b89d5648dca9f3aa9eadbf311a7729f30ca31520 Mon Sep 17 00:00:00 2001
From: aegilops <41705651+aegilops@users.noreply.github.com>
Date: Fri, 19 Apr 2024 11:30:02 +0100
Subject: [PATCH 08/95] Updated README.md

---
 README.md          |  4 +--
 database/README.md | 62 ++++++++++++++++++++++++++++++++++++++++++++++
 vendors/README.md  | 62 ----------------------------------------------
 3 files changed, 64 insertions(+), 64 deletions(-)

diff --git a/README.md b/README.md
index 52248e2d..bda7f473 100644
--- a/README.md
+++ b/README.md
@@ -45,6 +45,8 @@ Custom Secret Scanning Patterns repository.
 - SQLAlchemy Database Connection String
 
 - MongoDB Database Connection String
+
+- MongoDB connection string
   
 
 ### [Generic Secrets / Passwords](./generic)
@@ -171,8 +173,6 @@ Custom Secret Scanning Patterns repository.
 
 - Vercel OAuth client secrets
 
-- MongoDB connection string
-
 - UUIDv4 Bearer token (maybe Heroku)
 
 - Azure client secret
diff --git a/database/README.md b/database/README.md
index ba29833b..7be67499 100644
--- a/database/README.md
+++ b/database/README.md
@@ -326,4 +326,66 @@ Add these additional matches to the [Secret Scanning Custom Pattern](https://doc
   ^\$[A-Za-z_]+$
   ```
 
+</details>
+
+## MongoDB connection string
+
+
+
+_version: v0.2_
+
+
+
+<details>
+<summary>Pattern Format</summary>
+
+```regex
+mongodb(\+[a-z]+)?://[^'"`<>/:@\s\x00-\x08]+:[^'"`<>/@\s\x00-\x08]+@[^?'"`\s\x00-\x08]+
+```
+
+</details>
+
+<details>
+<summary>Start Pattern</summary>
+
+```regex
+\A|\b
+```
+
+</details><details>
+<summary>End Pattern</summary>
+
+```regex
+\z|\s|['"`?]
+```
+
+</details>
+
+<details>
+<summary>Additional Matches</summary>
+
+Add these additional matches to the [Secret Scanning Custom Pattern](https://docs.github.com/en/enterprise-cloud@latest/code-security/secret-scanning/defining-custom-patterns-for-secret-scanning#example-of-a-custom-pattern-specified-using-additional-requirements).
+
+
+- Not Match:
+
+  ```regex
+  (?i):(test|a|my)?[_-]?pass(word)?@
+  ```
+- Not Match:
+
+  ```regex
+  :%(?:\.\*)?[sv]@
+  ```
+- Not Match:
+
+  ```regex
+  :\$?\{[^}+]\}@
+  ```
+- Not Match:
+
+  ```regex
+  ^mongodb\+srv://b\*b%40f3tt%3D:%244to%40L8%3DMC@test3.test.build.10gen.cc/mydb%3F\?replicaSet=repl0
+  ```
+
 </details>
\ No newline at end of file
diff --git a/vendors/README.md b/vendors/README.md
index 73a68d3d..6c5038ca 100644
--- a/vendors/README.md
+++ b/vendors/README.md
@@ -963,68 +963,6 @@ _version: v0.1_
 
 </details>
 
-## MongoDB connection string
-
-
-
-_version: v0.1_
-
-
-
-<details>
-<summary>Pattern Format</summary>
-
-```regex
-mongodb\+srv://[^'"<>/:@\s\x00-\x08]+:[^'"<>/@\s\x00-\x08]+@[^/\s\x00-\x08]+\S*
-```
-
-</details>
-
-<details>
-<summary>Start Pattern</summary>
-
-```regex
-\A|\b
-```
-
-</details><details>
-<summary>End Pattern</summary>
-
-```regex
-\z|\s|['"`]
-```
-
-</details>
-
-<details>
-<summary>Additional Matches</summary>
-
-Add these additional matches to the [Secret Scanning Custom Pattern](https://docs.github.com/en/enterprise-cloud@latest/code-security/secret-scanning/defining-custom-patterns-for-secret-scanning#example-of-a-custom-pattern-specified-using-additional-requirements).
-
-
-- Not Match:
-
-  ```regex
-  ^mongodb\+srv://(test-)?user:(test-)?pass(word)?@
-  ```
-- Not Match:
-
-  ```regex
-  ^mongodb\+srv://%s:%s@
-  ```
-- Not Match:
-
-  ```regex
-  ^mongodb\+srv://auser:apass@
-  ```
-- Not Match:
-
-  ```regex
-  ^mongodb\+srv://b\*b%40f3tt%3D:%244to%40L8%3DMC@test3.test.build.10gen.cc/mydb%3F\?replicaSet=repl0
-  ```
-
-</details>
-
 ## UUIDv4 Bearer token (maybe Heroku)
 
 

From c8287df8edb328824c7bde1837c585eac4db4978 Mon Sep 17 00:00:00 2001
From: aegilops <41705651+aegilops@users.noreply.github.com>
Date: Fri, 19 Apr 2024 11:54:12 +0100
Subject: [PATCH 09/95] Updated MongoDB pattern, remove password-specific
 version

---
 database/patterns.yml | 16 ----------------
 1 file changed, 16 deletions(-)

diff --git a/database/patterns.yml b/database/patterns.yml
index accfee82..b532a18d 100644
--- a/database/patterns.yml
+++ b/database/patterns.yml
@@ -133,22 +133,6 @@ patterns:
       start_offset: 53
       end_offset: 58
 
-  - name: MongoDB Database Connection String
-    type: mongodb_connection_string
-    description: MongoDB connection strings are used to connect to databases, often with embedded credentials.
-    regex:
-      pattern: |
-        [^'"<>/@\s\x00-\x08]+
-      start: |
-        (\A|\b)mongodb(\+[a-z]+)?://[^'"<>/:@\s\x00-\x08]+:
-      end: |
-        @
-      additional_not_match:
-        - ^((test-?|a|my)?pass(word)?|%244to%40L8%3DMC)$
-        - ^%(?:\.\*)?[sv]$
-        - ^\$?\{[^}+]\}$
-        - ^\$[A-Za-z_]+$
-
   - name: MongoDB connection string
     type: mongodb_connection_string
     regex:

From a1d04fc881ebc172a9a7b0bb3f4e4b2becfaf584 Mon Sep 17 00:00:00 2001
From: aegilops <41705651+aegilops@users.noreply.github.com>
Date: Fri, 19 Apr 2024 11:54:20 +0100
Subject: [PATCH 10/95] Updated README.md

---
 README.md          |  2 --
 database/README.md | 62 ----------------------------------------------
 2 files changed, 64 deletions(-)

diff --git a/README.md b/README.md
index bda7f473..0d97159a 100644
--- a/README.md
+++ b/README.md
@@ -44,8 +44,6 @@ Custom Secret Scanning Patterns repository.
 
 - SQLAlchemy Database Connection String
 
-- MongoDB Database Connection String
-
 - MongoDB connection string
   
 
diff --git a/database/README.md b/database/README.md
index 7be67499..f7b8f0da 100644
--- a/database/README.md
+++ b/database/README.md
@@ -266,68 +266,6 @@ Add these additional matches to the [Secret Scanning Custom Pattern](https://doc
 
 </details>
 
-## MongoDB Database Connection String
-
-
-MongoDB connection strings are used to connect to databases, often with embedded credentials.
-_version: v0.1_
-
-
-
-<details>
-<summary>Pattern Format</summary>
-
-```regex
-[^'"<>/@\s\x00-\x08]+
-```
-
-</details>
-
-<details>
-<summary>Start Pattern</summary>
-
-```regex
-(\A|\b)mongodb(\+[a-z]+)?://[^'"<>/:@\s\x00-\x08]+:
-```
-
-</details><details>
-<summary>End Pattern</summary>
-
-```regex
-@
-```
-
-</details>
-
-<details>
-<summary>Additional Matches</summary>
-
-Add these additional matches to the [Secret Scanning Custom Pattern](https://docs.github.com/en/enterprise-cloud@latest/code-security/secret-scanning/defining-custom-patterns-for-secret-scanning#example-of-a-custom-pattern-specified-using-additional-requirements).
-
-
-- Not Match:
-
-  ```regex
-  ^((test-?|a|my)?pass(word)?|%244to%40L8%3DMC)$
-  ```
-- Not Match:
-
-  ```regex
-  ^%(?:\.\*)?[sv]$
-  ```
-- Not Match:
-
-  ```regex
-  ^\$?\{[^}+]\}$
-  ```
-- Not Match:
-
-  ```regex
-  ^\$[A-Za-z_]+$
-  ```
-
-</details>
-
 ## MongoDB connection string
 
 

From bfcbcb49c95b31827c33f694e107c6974d9d7fb8 Mon Sep 17 00:00:00 2001
From: aegilops <41705651+aegilops@users.noreply.github.com>
Date: Fri, 19 Apr 2024 17:02:45 +0100
Subject: [PATCH 11/95] Added AlienVault OTX API key

---
 vendors/patterns.yml | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/vendors/patterns.yml b/vendors/patterns.yml
index 1aca0e57..86c79f90 100644
--- a/vendors/patterns.yml
+++ b/vendors/patterns.yml
@@ -571,3 +571,19 @@ patterns:
         OPENSTACK_PASSWORD: "password1234isnotagoodpassword"
       start_offset: 21
       end_offset: 51
+
+  - name: AlienVault OTX API key
+    type: alienvault_otx_api_key
+    description: AlienVault OTX API key
+    regex:
+      pattern: |
+        [a-f0-9]{64}|[a-f0-9]{40}
+      start: |
+        (?i)ALIENVAULT(_?OTX)?(_?API)?_?KEY['"`]?(\s*[\]\)])?\s*([:,=]|[=-]>|to|[!=]={1,2}|<>)?\s*([[{])?['"`]?
+      end: |
+        ['"`\r\n,]|\z
+    test:
+      data: |
+        ALIENVAULT_OTX_API_KEY: "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
+      start_offset: 25
+      end_offset: 89

From febdba1e297b71674c93d26f6c76e4d1769e7be4 Mon Sep 17 00:00:00 2001
From: aegilops <41705651+aegilops@users.noreply.github.com>
Date: Fri, 19 Apr 2024 17:02:55 +0100
Subject: [PATCH 12/95] Updated README.md

---
 README.md         |  2 ++
 vendors/README.md | 33 +++++++++++++++++++++++++++++++++
 2 files changed, 35 insertions(+)

diff --git a/README.md b/README.md
index 0d97159a..b334ab73 100644
--- a/README.md
+++ b/README.md
@@ -178,4 +178,6 @@ Custom Secret Scanning Patterns repository.
 - Google private key id (or older API key)
 
 - OpenStack password/API key
+
+- AlienVault OTX API key
   
\ No newline at end of file
diff --git a/vendors/README.md b/vendors/README.md
index 6c5038ca..a78cec70 100644
--- a/vendors/README.md
+++ b/vendors/README.md
@@ -1154,4 +1154,37 @@ Add these additional matches to the [Secret Scanning Custom Pattern](https://doc
   ^(@?[a-z_]+\[:.*\]|@[a-z_]+)$
   ```
 
+</details>
+
+## AlienVault OTX API key
+
+
+AlienVault OTX API key
+_version: v0.1_
+
+
+
+<details>
+<summary>Pattern Format</summary>
+
+```regex
+[a-f0-9]{64}|[a-f0-9]{40}
+```
+
+</details>
+
+<details>
+<summary>Start Pattern</summary>
+
+```regex
+(?i)ALIENVAULT(_?OTX)?(_?API)?_?KEY['"`]?(\s*[\]\)])?\s*([:,=]|[=-]>|to|[!=]={1,2}|<>)?\s*([[{])?['"`]?
+```
+
+</details><details>
+<summary>End Pattern</summary>
+
+```regex
+['"`\r\n,]|\z
+```
+
 </details>
\ No newline at end of file

From 308d63561a22456240a6cefe0eed3604e32623e9 Mon Sep 17 00:00:00 2001
From: aegilops <41705651+aegilops@users.noreply.github.com>
Date: Fri, 19 Apr 2024 17:28:53 +0100
Subject: [PATCH 13/95] Added .env style passwords

---
 configs/patterns.yml | 27 +++++++++++++++++++++++++++
 1 file changed, 27 insertions(+)

diff --git a/configs/patterns.yml b/configs/patterns.yml
index 4a47f04d..7c96ef7c 100644
--- a/configs/patterns.yml
+++ b/configs/patterns.yml
@@ -187,3 +187,30 @@ patterns:
         <machineKey decryptionKey="ABAA84D7EC4BB56D75D217CECFFB9628809BDB8BF91CFCD64568A145BE59719F" decryption="AES" />
       start_offset: 27
       end_offset: 91
+
+  - name: .env file style secrets
+    type: env_passwords
+    description: |
+      Find .env file style secrets in configuration files
+    experimental: true
+    regex:
+      version: 0.1
+      pattern: |
+        [^\r\n\x00-\x08'"#]+
+      start: |
+        (?:\n|\A)(ENV |export )?[A-Z_]*(?:SECRET|SERVICE_PASS(WD|WORD|CODE|PHRASE)|PASS(?:WD|WORD|CODE|PHRASE)?|KEY)=['"]?
+      end: |
+        ['"\r\n#]|\z
+      additional_not_match:
+        - ^\$\[{(]
+        - ^<[^>]+>$
+    comments:
+      - "Looks for secrets in the format of `SECRET=secret` at the start of a line, possibly with an `ENV ` or `export ` prefix"
+      - "Some false positives in code might appear"
+      - "The pattern only checks for certain key words to begin the pattern (`secret`, `password`, etc.)"
+      - "More restrictive than the Generic Passwords pattern, so less prone to false positives"
+    test:
+      data: |
+        SECRET_KEY=password123
+      start_offset: 11
+      end_offset: 22

From 37c0587dad07c6462676457d36b11cd545454d77 Mon Sep 17 00:00:00 2001
From: aegilops <41705651+aegilops@users.noreply.github.com>
Date: Fri, 19 Apr 2024 17:40:08 +0100
Subject: [PATCH 14/95] Added Apollo.io API key

---
 vendors/patterns.yml | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/vendors/patterns.yml b/vendors/patterns.yml
index 86c79f90..50f686b4 100644
--- a/vendors/patterns.yml
+++ b/vendors/patterns.yml
@@ -587,3 +587,19 @@ patterns:
         ALIENVAULT_OTX_API_KEY: "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
       start_offset: 25
       end_offset: 89
+
+  - name: Apollo.io API key
+    type: apollo_io_api_key
+    description: Apollo.io API key
+    regex:
+      pattern: |
+        service:[A-Za-z0-9-]+:\S+
+      start: |
+        (?i)key['"`]?(\s*[\]\)])?\s*([:,=]|[=-]>|to|[!=]={1,2}|<>)?\s*([[{])?['"`]?
+      end: |
+        ['"`,]|\z|\s
+    test:
+      data: |
+        key: service:my-apollo-service:aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa
+      start_offset: 5
+      end_offset: 71

From 9c51b746d8c9de55be89d90f12c0d586a4fec995 Mon Sep 17 00:00:00 2001
From: aegilops <41705651+aegilops@users.noreply.github.com>
Date: Fri, 19 Apr 2024 18:25:33 +0100
Subject: [PATCH 15/95] Added Jenkins API token

---
 vendors/patterns.yml | 48 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 48 insertions(+)

diff --git a/vendors/patterns.yml b/vendors/patterns.yml
index 50f686b4..fdcc5ada 100644
--- a/vendors/patterns.yml
+++ b/vendors/patterns.yml
@@ -603,3 +603,51 @@ patterns:
         key: service:my-apollo-service:aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa
       start_offset: 5
       end_offset: 71
+
+  - name: ClickUp API key
+    type: clickup_api_key
+    description: ClickUp API key
+    regex:
+      pattern: |
+        pk_[0-9]{6,8}_[A-Z0-9]{32}
+      start: |
+        \b|\A
+      end: |
+        \b|\z
+    test:
+      data: |
+        CLICKUP_API_KEY: "pk_1234567_ABCDEFGHIJKLMNOPQRSTUVWXYZ123456"
+      start_offset: 18
+      end_offset: 61
+
+  - name: Amazon MWS Auth Token
+    type: amazon_mws_auth_token
+    description: Amazon MWS Auth Token
+    regex:
+      pattern: |
+        amzn\.mws\.[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}
+      start: |
+        \b|\A
+      end: |
+        \b|\z
+    test:
+      data: |
+        amzn.mws.auth.token: "amzn.mws.12345678-1234-1234-1234-1234567890ab"
+      start_offset: 22
+      end_offset: 67
+
+  - name: Jenkins API token
+    type: jenkins_api_token
+    description: Jenkins API token
+    regex:
+      pattern: |
+        [a-f0-9]{32}
+      start: |
+        (?i)jenkins(api[_-]?)?(token|secret|key)['"`]?(\s*[\]\)])?\s*([:,=]|[=-]>|to|[!=]={1,2}|<>)?\s*([[{])?['"`]?
+      end: |
+        ['"`\r\n,]|\z
+    test:
+      data: |
+        jenkins_api_token="0123456789abcdef0123456789abcdef"
+      start_offset: 19
+      end_offset: 52

From 1239c0a96251931c842698a895a15107a6d6c850 Mon Sep 17 00:00:00 2001
From: aegilops <41705651+aegilops@users.noreply.github.com>
Date: Fri, 19 Apr 2024 18:25:46 +0100
Subject: [PATCH 16/95] Updated README.md

---
 README.md         |   8 +++
 configs/README.md |  63 ++++++++++++++++++++++
 vendors/README.md | 132 ++++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 203 insertions(+)

diff --git a/README.md b/README.md
index b334ab73..383e8abb 100644
--- a/README.md
+++ b/README.md
@@ -180,4 +180,12 @@ Custom Secret Scanning Patterns repository.
 - OpenStack password/API key
 
 - AlienVault OTX API key
+
+- Apollo.io API key
+
+- ClickUp API key
+
+- Amazon MWS Auth Token
+
+- Jenkins API token
   
\ No newline at end of file
diff --git a/configs/README.md b/configs/README.md
index a7295cea..6d4ab173 100644
--- a/configs/README.md
+++ b/configs/README.md
@@ -320,4 +320,67 @@ _version: v0.1_
 \"
 ```
 
+</details>
+
+## .env file style secrets
+
+**⚠️ WARNING: THIS RULE IS EXPERIMENTAL AND MIGHT CAUSE A HIGH FALSE POSITIVE RATE (test before commiting to org level) ⚠️**
+Find .env file style secrets in configuration files
+
+_version: v0.1_
+
+**Comments / Notes:**
+
+
+- Looks for secrets in the format of `SECRET=secret` at the start of a line, possibly with an `ENV ` or `export ` prefix
+
+- Some false positives in code might appear
+
+- The pattern only checks for certain key words to begin the pattern (`secret`, `password`, etc.)
+
+- More restrictive than the Generic Passwords pattern, so less prone to false positives
+  
+
+<details>
+<summary>Pattern Format</summary>
+
+```regex
+[^\r\n\x00-\x08'"#]+
+```
+
+</details>
+
+<details>
+<summary>Start Pattern</summary>
+
+```regex
+(?:\n|\A)(ENV |export )?[A-Z_]*(?:SECRET|SERVICE_PASS(WD|WORD|CODE|PHRASE)|PASS(?:WD|WORD|CODE|PHRASE)?|KEY)=['"]?
+```
+
+</details><details>
+<summary>End Pattern</summary>
+
+```regex
+['"\r\n#]|\z
+```
+
+</details>
+
+<details>
+<summary>Additional Matches</summary>
+
+Add these additional matches to the [Secret Scanning Custom Pattern](https://docs.github.com/en/enterprise-cloud@latest/code-security/secret-scanning/defining-custom-patterns-for-secret-scanning#example-of-a-custom-pattern-specified-using-additional-requirements).
+
+
+- Not Match:
+
+  ```regex
+  ^\$\[{(]
+  ```
+- Not Match:
+
+  ```regex
+  ^<[^>]+>$
+  ```
+
 </details>
\ No newline at end of file
diff --git a/vendors/README.md b/vendors/README.md
index a78cec70..ac3255af 100644
--- a/vendors/README.md
+++ b/vendors/README.md
@@ -1187,4 +1187,136 @@ _version: v0.1_
 ['"`\r\n,]|\z
 ```
 
+</details>
+
+## Apollo.io API key
+
+
+Apollo.io API key
+_version: v0.1_
+
+
+
+<details>
+<summary>Pattern Format</summary>
+
+```regex
+service:[A-Za-z0-9-]+:\S+
+```
+
+</details>
+
+<details>
+<summary>Start Pattern</summary>
+
+```regex
+(?i)key['"`]?(\s*[\]\)])?\s*([:,=]|[=-]>|to|[!=]={1,2}|<>)?\s*([[{])?['"`]?
+```
+
+</details><details>
+<summary>End Pattern</summary>
+
+```regex
+['"`,]|\z|\s
+```
+
+</details>
+
+## ClickUp API key
+
+
+ClickUp API key
+_version: v0.1_
+
+
+
+<details>
+<summary>Pattern Format</summary>
+
+```regex
+pk_[0-9]{6,8}_[A-Z0-9]{32}
+```
+
+</details>
+
+<details>
+<summary>Start Pattern</summary>
+
+```regex
+\b|\A
+```
+
+</details><details>
+<summary>End Pattern</summary>
+
+```regex
+\b|\z
+```
+
+</details>
+
+## Amazon MWS Auth Token
+
+
+Amazon MWS Auth Token
+_version: v0.1_
+
+
+
+<details>
+<summary>Pattern Format</summary>
+
+```regex
+amzn\.mws\.[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}
+```
+
+</details>
+
+<details>
+<summary>Start Pattern</summary>
+
+```regex
+\b|\A
+```
+
+</details><details>
+<summary>End Pattern</summary>
+
+```regex
+\b|\z
+```
+
+</details>
+
+## Jenkins API token
+
+
+Jenkins API token
+_version: v0.1_
+
+
+
+<details>
+<summary>Pattern Format</summary>
+
+```regex
+[a-f0-9]{32}
+```
+
+</details>
+
+<details>
+<summary>Start Pattern</summary>
+
+```regex
+(?i)jenkins(api[_-]?)?(token|secret|key)['"`]?(\s*[\]\)])?\s*([:,=]|[=-]>|to|[!=]={1,2}|<>)?\s*([[{])?['"`]?
+```
+
+</details><details>
+<summary>End Pattern</summary>
+
+```regex
+['"`\r\n,]|\z
+```
+
 </details>
\ No newline at end of file

From fb7ec23235019f6415052546dc3c452f298b5dcc Mon Sep 17 00:00:00 2001
From: aegilops <41705651+aegilops@users.noreply.github.com>
Date: Fri, 19 Apr 2024 18:35:56 +0100
Subject: [PATCH 17/95] Fixed Jenkins pattern

---
 vendors/patterns.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/vendors/patterns.yml b/vendors/patterns.yml
index fdcc5ada..f16d61e6 100644
--- a/vendors/patterns.yml
+++ b/vendors/patterns.yml
@@ -641,9 +641,9 @@ patterns:
     description: Jenkins API token
     regex:
       pattern: |
-        [a-f0-9]{32}
+        [a-f0-9]{32,64}
       start: |
-        (?i)jenkins(api[_-]?)?(token|secret|key)['"`]?(\s*[\]\)])?\s*([:,=]|[=-]>|to|[!=]={1,2}|<>)?\s*([[{])?['"`]?
+        (?i)jenkins_?(api[_-]?)?(token|secret|key)['"`]?(\s*[\]\)])?\s*([:,=]|[=-]>|to|[!=]={1,2}|<>)?\s*([[{])?['"`]?
       end: |
         ['"`\r\n,]|\z
     test:

From 3291e5a17980b7aa6c8cf9a53d956eceb43f3a7f Mon Sep 17 00:00:00 2001
From: aegilops <41705651+aegilops@users.noreply.github.com>
Date: Fri, 19 Apr 2024 18:36:01 +0100
Subject: [PATCH 18/95] Updated README.md

---
 vendors/README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/vendors/README.md b/vendors/README.md
index ac3255af..cd4a6c36 100644
--- a/vendors/README.md
+++ b/vendors/README.md
@@ -1300,7 +1300,7 @@ _version: v0.1_
 <summary>Pattern Format</summary>
 
 ```regex
-[a-f0-9]{32}
+[a-f0-9]{32,64}
 ```
 
 </details>
@@ -1309,7 +1309,7 @@ _version: v0.1_
 <summary>Start Pattern</summary>
 
 ```regex
-(?i)jenkins(api[_-]?)?(token|secret|key)['"`]?(\s*[\]\)])?\s*([:,=]|[=-]>|to|[!=]={1,2}|<>)?\s*([[{])?['"`]?
+(?i)jenkins_?(api[_-]?)?(token|secret|key)['"`]?(\s*[\]\)])?\s*([:,=]|[=-]>|to|[!=]={1,2}|<>)?\s*([[{])?['"`]?
 ```
 
 </details><details>

From 385785b482458be26fea7aa69429aa91128a645d Mon Sep 17 00:00:00 2001
From: aegilops <41705651+aegilops@users.noreply.github.com>
Date: Fri, 19 Apr 2024 19:14:56 +0100
Subject: [PATCH 19/95] Database connection string as full secret

---
 database/patterns.yml | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/database/patterns.yml b/database/patterns.yml
index b532a18d..ec92f570 100644
--- a/database/patterns.yml
+++ b/database/patterns.yml
@@ -8,6 +8,22 @@ name: Database passwords
 # We don't account for connection strings encoded in UTF-16, but that will be a rare case, if it happens at all
 
 patterns:
+  - name: Database Connection String (full string)
+    type: database_connection_string_full
+    description: "Database connection strings are used to connect to databases, often with embedded credentials."
+    regex:
+      pattern: |
+        [^"'`\x00-\x08\r\n|]*([Ss]erver|[Pp]rovider|[Dd]atabase|[Uu]ser [Ii]d|[Dd]ata [Ss]ource|[Ee]ndpoint|[Dd]efault[Ee]nd[Pp]oints[Pp]rotocol|[Aa]ccountName|[Da]ata[Ss]ource|[Aa]uthentication|[Ll]ogin|[Ii]nitial[Cc]atalog|DB|Trusted_Connection|authenticationType|DSN|[Dd]ata[Ss]ource[Nn]ame|[Ii]ntegrated[Ss]ecurity|[Ll]ocation|[Ee]ncrypt|[Ss]ystem|[Pp]rotocol|[Hh]ost|[Pp]ort|SRVR|[Dd]river|Dbq|[Ss]sl[Mm]ode|SSL|[Uu]id|DBNAME|SystemDB|[Pp]ersist [Ss]ecurity [Ii]nfo|[Cc]onnection [Tt]ype|[Dd]ata[Ss]ource[Nn]ame|[Ee]xcel [Ff]ile|[Ss]erver [Nn]ame|URL)=[^"'`\x00-\x08\r\n|]*
+      start: |
+        \A|["'`]|&quot;|[\r\n]|[=:-] 
+      end: |
+        \z|["'`]|&quot;|[\r\n]
+      additional_match:
+        - (^|;)([Pp]assword|[Pp]wd|[Ss]hared[Ss]ecret[Vv]alue|[Aa]ccount[Kk]ey|PW|pw|[Cc]ipher [Kk]ey|OAuth Access Token Secret)=
+      additional_not_match:
+        # placeholders, variable substitutions, etc.
+        - (^|;)([Pp]assword|[Pp]wd|[Ss]hared[Ss]ecret[Vv]alue|[Aa]ccount[Kk]ey|PW|pw|[Cc]ipher [Kk]ey|OAuth Access Token Secret)=(%s\b|\{\{[^}]+\}\}|\{[0-9]+\}|\$?\{[^}]+\}|\[[A-Z_]+\]|['"`]|$)
+
   - name: Database Connection String (1)
     type: database_connection_string_1
     description: "Database connection strings are used to connect to databases, often with embedded credentials."

From 641f9f2fe45769ece66c1f23e8cf08297ebc6125 Mon Sep 17 00:00:00 2001
From: aegilops <41705651+aegilops@users.noreply.github.com>
Date: Fri, 19 Apr 2024 19:15:04 +0100
Subject: [PATCH 20/95] Updated README.md

---
 README.md          |  4 ++++
 database/README.md | 53 ++++++++++++++++++++++++++++++++++++++++++++++
 vendors/README.md  | 33 +++++++++++++++++++++++++++++
 3 files changed, 90 insertions(+)

diff --git a/README.md b/README.md
index 383e8abb..48d9cd18 100644
--- a/README.md
+++ b/README.md
@@ -34,6 +34,8 @@ Custom Secret Scanning Patterns repository.
 
 
 
+- Database Connection String (full string)
+
 - Database Connection String (1)
 
 - Database Connection String (2)
@@ -188,4 +190,6 @@ Custom Secret Scanning Patterns repository.
 - Amazon MWS Auth Token
 
 - Jenkins API token
+
+- AWS S3 presigned URL
   
\ No newline at end of file
diff --git a/database/README.md b/database/README.md
index f7b8f0da..3b166a84 100644
--- a/database/README.md
+++ b/database/README.md
@@ -5,6 +5,59 @@
 
 # Database passwords
 
+## Database Connection String (full string)
+
+
+Database connection strings are used to connect to databases, often with embedded credentials.
+_version: v0.1_
+
+
+
+<details>
+<summary>Pattern Format</summary>
+
+```regex
+[^"'`\x00-\x08\r\n|]*([Ss]erver|[Pp]rovider|[Dd]atabase|[Uu]ser [Ii]d|[Dd]ata [Ss]ource|[Ee]ndpoint|[Dd]efault[Ee]nd[Pp]oints[Pp]rotocol|[Aa]ccountName|[Da]ata[Ss]ource|[Aa]uthentication|[Ll]ogin|[Ii]nitial[Cc]atalog|DB|Trusted_Connection|authenticationType|DSN|[Dd]ata[Ss]ource[Nn]ame|[Ii]ntegrated[Ss]ecurity|[Ll]ocation|[Ee]ncrypt|[Ss]ystem|[Pp]rotocol|[Hh]ost|[Pp]ort|SRVR|[Dd]river|Dbq|[Ss]sl[Mm]ode|SSL|[Uu]id|DBNAME|SystemDB|[Pp]ersist [Ss]ecurity [Ii]nfo|[Cc]onnection [Tt]ype|[Dd]ata[Ss]ource[Nn]ame|[Ee]xcel [Ff]ile|[Ss]erver [Nn]ame|URL)=[^"'`\x00-\x08\r\n|]*
+```
+
+</details>
+
+<details>
+<summary>Start Pattern</summary>
+
+```regex
+\A|["'`]|&quot;|[\r\n]|[=:-]
+```
+
+</details><details>
+<summary>End Pattern</summary>
+
+```regex
+\z|["'`]|&quot;|[\r\n]
+```
+
+</details>
+
+<details>
+<summary>Additional Matches</summary>
+
+Add these additional matches to the [Secret Scanning Custom Pattern](https://docs.github.com/en/enterprise-cloud@latest/code-security/secret-scanning/defining-custom-patterns-for-secret-scanning#example-of-a-custom-pattern-specified-using-additional-requirements).
+
+
+
+- Match:
+
+  ```regex
+  (^|;)([Pp]assword|[Pp]wd|[Ss]hared[Ss]ecret[Vv]alue|[Aa]ccount[Kk]ey|PW|pw|[Cc]ipher [Kk]ey|OAuth Access Token Secret)=
+  ```
+- Not Match:
+
+  ```regex
+  (^|;)([Pp]assword|[Pp]wd|[Ss]hared[Ss]ecret[Vv]alue|[Aa]ccount[Kk]ey|PW|pw|[Cc]ipher [Kk]ey|OAuth Access Token Secret)=(%s\b|\{\{[^}]+\}\}|\{[0-9]+\}|\$?\{[^}]+\}|\[[A-Z_]+\]|['"`]|$)
+  ```
+
+</details>
+
 ## Database Connection String (1)
 
 
diff --git a/vendors/README.md b/vendors/README.md
index cd4a6c36..97b5b9dc 100644
--- a/vendors/README.md
+++ b/vendors/README.md
@@ -1319,4 +1319,37 @@ _version: v0.1_
 ['"`\r\n,]|\z
 ```
 
+</details>
+
+## AWS S3 presigned URL
+
+
+AWS S3 presigned URL
+_version: v0.1_
+
+
+
+<details>
+<summary>Pattern Format</summary>
+
+```regex
+https://[a-z-]+\.s3\.amazonaws\.com/[^?\s'"`\r\n]+\?[^\s'"`\r\n]+&X-Amz-Signature=[^\s'"`\r\n]+
+```
+
+</details>
+
+<details>
+<summary>Start Pattern</summary>
+
+```regex
+\b|\A
+```
+
+</details><details>
+<summary>End Pattern</summary>
+
+```regex
+['"`\r\n,]|\z
+```
+
 </details>
\ No newline at end of file

From baa42e78848fa0d432b363fe1f41c526c1aa3d57 Mon Sep 17 00:00:00 2001
From: aegilops <41705651+aegilops@users.noreply.github.com>
Date: Fri, 19 Apr 2024 19:18:21 +0100
Subject: [PATCH 21/95] Improved Apollo API key pattern

---
 vendors/patterns.yml | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/vendors/patterns.yml b/vendors/patterns.yml
index f16d61e6..b19eb2f3 100644
--- a/vendors/patterns.yml
+++ b/vendors/patterns.yml
@@ -651,3 +651,17 @@ patterns:
         jenkins_api_token="0123456789abcdef0123456789abcdef"
       start_offset: 19
       end_offset: 52
+
+  - name: AWS S3 presigned URL
+    type: aws_s3_presigned_url
+    description: AWS S3 presigned URL
+    regex:
+      pattern: |
+        https://[a-z-]+\.s3\.amazonaws\.com/[^?\s'"`\r\n]+\?[^\s'"`\r\n]+&X-Amz-Signature=[^\s'"`\r\n]+
+      start: |
+        \b|\A
+      end: |
+        ['"`\r\n,]|\z
+    test:
+      data: |
+        https://snort-org-site.s3.amazonaws.com/production/document_files/files/000/000/596/original/Rules_Writers_Guide_to_Snort_3_Rules.pdf?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAIXACIED2SPMSC7GA%2F20211011%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20211011T165700Z&X-Amz-Expires=172800&X-Amz-SignedHeaders=host&X-Amz-Signature=1d5134d3a84db0f7b1ece95fcf98387a0cc32ff131b6c95e6a80d3cd4b616a5c

From 62a8e2d203511a9aa6f4d0e78844762e1596c511 Mon Sep 17 00:00:00 2001
From: aegilops <41705651+aegilops@users.noreply.github.com>
Date: Mon, 22 Apr 2024 12:23:39 +0100
Subject: [PATCH 22/95] Trying to fix .env pattern

---
 configs/patterns.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/configs/patterns.yml b/configs/patterns.yml
index 7c96ef7c..2d2c91ea 100644
--- a/configs/patterns.yml
+++ b/configs/patterns.yml
@@ -202,7 +202,7 @@ patterns:
       end: |
         ['"\r\n#]|\z
       additional_not_match:
-        - ^\$\[{(]
+        - ^\$[{(]
         - ^<[^>]+>$
     comments:
       - "Looks for secrets in the format of `SECRET=secret` at the start of a line, possibly with an `ENV ` or `export ` prefix"

From 2321b07ad8e9018349b1ce10628116e6a0c5a834 Mon Sep 17 00:00:00 2001
From: aegilops <41705651+aegilops@users.noreply.github.com>
Date: Tue, 23 Apr 2024 11:16:46 +0100
Subject: [PATCH 23/95] Legacy Azure access key

---
 vendors/patterns.yml | 24 +++++++++++++++++++++++-
 1 file changed, 23 insertions(+), 1 deletion(-)

diff --git a/vendors/patterns.yml b/vendors/patterns.yml
index b19eb2f3..08380299 100644
--- a/vendors/patterns.yml
+++ b/vendors/patterns.yml
@@ -593,7 +593,7 @@ patterns:
     description: Apollo.io API key
     regex:
       pattern: |
-        service:[A-Za-z0-9-]+:\S+
+        service:[A-Za-z0-9-]+:[^\s'"`,\x00-\x08\x7f-\xff]+
       start: |
         (?i)key['"`]?(\s*[\]\)])?\s*([:,=]|[=-]>|to|[!=]={1,2}|<>)?\s*([[{])?['"`]?
       end: |
@@ -665,3 +665,25 @@ patterns:
     test:
       data: |
         https://snort-org-site.s3.amazonaws.com/production/document_files/files/000/000/596/original/Rules_Writers_Guide_to_Snort_3_Rules.pdf?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAIXACIED2SPMSC7GA%2F20211011%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20211011T165700Z&X-Amz-Expires=172800&X-Amz-SignedHeaders=host&X-Amz-Signature=1d5134d3a84db0f7b1ece95fcf98387a0cc32ff131b6c95e6a80d3cd4b616a5c
+
+  - name: Azure Access Key (legacy format)
+    type: azure_access_key_legacy
+    description: Azure Access Key in context in a variable assignment - legacy key format without internal identifiable features
+    regex:
+      pattern: |
+        [A-Za-z0-9/+]+==
+      start: |
+        (\A|\b)(?i)(AZURE|ACCOUNT)(_?ACCESS|_?STORAGE(_?ACCOUNT)?)?_?KEY['"`]?(\s*[\]\)])?\s*([:,=]|[=-]>|to|[!=]={1,2}|<>)?\s*([[{])?['"`]?
+      end: |
+        ['"`\r\n,]|\z
+      additional_match:
+        - ^[A-Za-z0-9/+]{86}==$
+    test:
+      data: |
+        AZURE_ACCESS_KEY: "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789abcdefghijklmn=="
+      start_offset: 19
+      end_offset: 107
+    comments:
+      - This is a legacy format for Azure Access Keys. The key is base64 encoded and encodes a fixed length key, so we know its length and that it always end in `==`.
+      - The key lacks internal identifiable features, which are used in modern keys issued by these Azure services.
+      - The use of `+` instead of `{86}` in the regex pattern is due to limitations of secret scanning - make sure you use the "additional match" to constrain the length

From 8fb2f05cc19d7a52475312330f26f7d2b59001e0 Mon Sep 17 00:00:00 2001
From: aegilops <41705651+aegilops@users.noreply.github.com>
Date: Tue, 23 Apr 2024 11:18:51 +0100
Subject: [PATCH 24/95] Updated README.md

---
 README.md         |  2 ++
 configs/README.md |  2 +-
 vendors/README.md | 58 ++++++++++++++++++++++++++++++++++++++++++++++-
 3 files changed, 60 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 48d9cd18..3ab9bd54 100644
--- a/README.md
+++ b/README.md
@@ -192,4 +192,6 @@ Custom Secret Scanning Patterns repository.
 - Jenkins API token
 
 - AWS S3 presigned URL
+
+- Azure Access Key (legacy format)
   
\ No newline at end of file
diff --git a/configs/README.md b/configs/README.md
index 6d4ab173..6557df1d 100644
--- a/configs/README.md
+++ b/configs/README.md
@@ -375,7 +375,7 @@ Add these additional matches to the [Secret Scanning Custom Pattern](https://doc
 - Not Match:
 
   ```regex
-  ^\$\[{(]
+  ^\$[{(]
   ```
 - Not Match:
 
diff --git a/vendors/README.md b/vendors/README.md
index 97b5b9dc..144d0d08 100644
--- a/vendors/README.md
+++ b/vendors/README.md
@@ -1201,7 +1201,7 @@ _version: v0.1_
 <summary>Pattern Format</summary>
 
 ```regex
-service:[A-Za-z0-9-]+:\S+
+service:[A-Za-z0-9-]+:[^\s'"`,\x00-\x08\x7f-\xff]+
 ```
 
 </details>
@@ -1352,4 +1352,60 @@ https://[a-z-]+\.s3\.amazonaws\.com/[^?\s'"`\r\n]+\?[^\s'"`\r\n]+&X-Amz-Signatur
 ['"`\r\n,]|\z
 ```
 
+</details>
+
+## Azure Access Key (legacy format)
+
+
+Azure Access Key in context in a variable assignment - legacy key format without internal identifiable features
+_version: v0.1_
+
+**Comments / Notes:**
+
+
+- This is a legacy format for Azure Access Keys. The key is base64 encoded and encodes a fixed length key, so we know its length and that it always end in `==`.
+
+- The key lacks internal identifiable features, which are used in modern keys issued by these Azure services.
+
+- The use of `+` instead of `{86}` in the regex pattern is due to limitations of secret scanning - make sure you use the "additional match" to constrain the length
+  
+
+<details>
+<summary>Pattern Format</summary>
+
+```regex
+[A-Za-z0-9/+]+==
+```
+
+</details>
+
+<details>
+<summary>Start Pattern</summary>
+
+```regex
+(\A|\b)(?i)(AZURE|ACCOUNT)(_?ACCESS|_?STORAGE(_?ACCOUNT)?)?_?KEY['"`]?(\s*[\]\)])?\s*([:,=]|[=-]>|to|[!=]={1,2}|<>)?\s*([[{])?['"`]?
+```
+
+</details><details>
+<summary>End Pattern</summary>
+
+```regex
+['"`\r\n,]|\z
+```
+
+</details>
+
+<details>
+<summary>Additional Matches</summary>
+
+Add these additional matches to the [Secret Scanning Custom Pattern](https://docs.github.com/en/enterprise-cloud@latest/code-security/secret-scanning/defining-custom-patterns-for-secret-scanning#example-of-a-custom-pattern-specified-using-additional-requirements).
+
+
+
+- Match:
+
+  ```regex
+  ^[A-Za-z0-9/+]{86}==$
+  ```
+
 </details>
\ No newline at end of file

From aff2515ae270111eb80c3e2702c84d2e9f9cd866 Mon Sep 17 00:00:00 2001
From: aegilops <41705651+aegilops@users.noreply.github.com>
Date: Tue, 23 Apr 2024 12:19:54 +0100
Subject: [PATCH 25/95] Added Azure Shared Access Signature

---
 vendors/patterns.yml | 28 ++++++++++++++++++++++++++++
 1 file changed, 28 insertions(+)

diff --git a/vendors/patterns.yml b/vendors/patterns.yml
index 08380299..beada83e 100644
--- a/vendors/patterns.yml
+++ b/vendors/patterns.yml
@@ -687,3 +687,31 @@ patterns:
       - This is a legacy format for Azure Access Keys. The key is base64 encoded and encodes a fixed length key, so we know its length and that it always end in `==`.
       - The key lacks internal identifiable features, which are used in modern keys issued by these Azure services.
       - The use of `+` instead of `{86}` in the regex pattern is due to limitations of secret scanning - make sure you use the "additional match" to constrain the length
+
+  - name: Azure Shared Access Signature (SAS) Token
+    type: azure_sas_token
+    description: Azure Shared Access Signature (SAS) Token
+    regex:
+      pattern: |
+        (https://[^?]+\?)?[^\s?/]*\bsig=([A-Za-z0-9]|%2[bfBF]){43,}%3[dD][^\s?/]*
+      start: |
+        \b|\A
+      end: |
+        \z|\s|['"`]
+      additional_match:
+        - '[?&]sv=[0-9]{4}-[0-9]{2}-[0-9]{2}'
+        - '[?&]se=[0-9]{4}-[0-9]{2}-[0-9]{2}'
+        - '[?&]st=[0-9]{4}-[0-9]{2}-[0-9]{2}'
+      additional_not_match:
+        - '^https://files\.oaiusercontent\.com/'
+    test:
+      data: |
+        AZURE_SAS_TOKEN: "?sv=2019-12-12&ss=bfqt&srt=sco&sp=rwdlacupx&se=2021-10-11T16:57:00Z&st=2021-10-11T08:57:00Z&spr=https&sig=abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789abcdefghijklmn%3D"
+      start_offset: 18
+      end_offset: 169
+    comments:
+      - This is a Shared Access Signature (SAS) token for Azure services. See [these examples](https://learn.microsoft.com/en-us/rest/api/storageservices/service-sas-examples)
+      - The token is a URL query string parameter, and the signature is a base64 encoded HMAC-SHA256 hash, so is a fixed length in plain text and always ends in =
+      - When encoded in a URL, the `+` character is replaced with `%2B`, the `/` character is replaced with `%2F`, and the `=` character is replaced with `%3D`
+      - Because of the variable length of the characters (beacuse of the URL encoding), we use `{43,}` to match the signature
+      - We ignore `https://files.oaiusercontent.com/` because they are URLs for images generated by ChatGPT

From 48b0dc0b3ac9f9f354bedd16f07bacfd50531cbc Mon Sep 17 00:00:00 2001
From: aegilops <41705651+aegilops@users.noreply.github.com>
Date: Tue, 23 Apr 2024 12:20:11 +0100
Subject: [PATCH 26/95] Updated README.md

---
 README.md         |  2 ++
 vendors/README.md | 77 +++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 79 insertions(+)

diff --git a/README.md b/README.md
index 3ab9bd54..35ea0390 100644
--- a/README.md
+++ b/README.md
@@ -194,4 +194,6 @@ Custom Secret Scanning Patterns repository.
 - AWS S3 presigned URL
 
 - Azure Access Key (legacy format)
+
+- Azure Shared Access Signature (SAS) Token
   
\ No newline at end of file
diff --git a/vendors/README.md b/vendors/README.md
index 144d0d08..4156a569 100644
--- a/vendors/README.md
+++ b/vendors/README.md
@@ -1408,4 +1408,81 @@ Add these additional matches to the [Secret Scanning Custom Pattern](https://doc
   ^[A-Za-z0-9/+]{86}==$
   ```
 
+</details>
+
+## Azure Shared Access Signature (SAS) Token
+
+
+Azure Shared Access Signature (SAS) Token
+_version: v0.1_
+
+**Comments / Notes:**
+
+
+- This is a Shared Access Signature (SAS) token for Azure services. See [these examples](https://learn.microsoft.com/en-us/rest/api/storageservices/service-sas-examples)
+
+- The token is a URL query string parameter, and the signature is a base64 encoded HMAC-SHA256 hash, so is a fixed length in plain text and always ends in =
+
+- When encoded in a URL, the `+` character is replaced with `%2B`, the `/` character is replaced with `%2F`, and the `=` character is replaced with `%3D`
+
+- Because of the variable length of the characters (beacuse of the URL encoding), we use `{43,}` to match the signature
+
+- We ignore `https://files.oaiusercontent.com/` because they are URLs for images generated by ChatGPT
+  
+
+<details>
+<summary>Pattern Format</summary>
+
+```regex
+(https://[^?]+\?)?[^\s?/]*\bsig=([A-Za-z0-9]|%2[bfBF]){43,}%3[dD][^\s?/]*
+```
+
+</details>
+
+<details>
+<summary>Start Pattern</summary>
+
+```regex
+\b|\A
+```
+
+</details><details>
+<summary>End Pattern</summary>
+
+```regex
+\z|\s|['"`]
+```
+
+</details>
+
+<details>
+<summary>Additional Matches</summary>
+
+Add these additional matches to the [Secret Scanning Custom Pattern](https://docs.github.com/en/enterprise-cloud@latest/code-security/secret-scanning/defining-custom-patterns-for-secret-scanning#example-of-a-custom-pattern-specified-using-additional-requirements).
+
+
+
+- Match:
+
+  ```regex
+  [?&]sv=[0-9]{4}-[0-9]{2}-[0-9]{2}
+  ```
+
+- Match:
+
+  ```regex
+  [?&]se=[0-9]{4}-[0-9]{2}-[0-9]{2}
+  ```
+
+- Match:
+
+  ```regex
+  [?&]st=[0-9]{4}-[0-9]{2}-[0-9]{2}
+  ```
+- Not Match:
+
+  ```regex
+  ^https://files\.oaiusercontent\.com/
+  ```
+
 </details>
\ No newline at end of file

From 9d5f6710f95086162d534c69d17d3dae6514c2a6 Mon Sep 17 00:00:00 2001
From: aegilops <41705651+aegilops@users.noreply.github.com>
Date: Mon, 29 Apr 2024 17:41:10 +0100
Subject: [PATCH 27/95] Added JDBC pattern

---
 database/patterns.yml | 18 +++++++++++++++++-
 1 file changed, 17 insertions(+), 1 deletion(-)

diff --git a/database/patterns.yml b/database/patterns.yml
index ec92f570..b90bce3e 100644
--- a/database/patterns.yml
+++ b/database/patterns.yml
@@ -165,4 +165,20 @@ patterns:
         - :\$?\{[^}+]\}@
         - ^mongodb\+srv://b\*b%40f3tt%3D:%244to%40L8%3DMC@test3.test.build.10gen.cc/mydb%3F\?replicaSet=repl0
     test:
-      data: mongodb+srv://foo:bar@mongodb.example.invalid/
\ No newline at end of file
+      data: mongodb+srv://foo:bar@mongodb.example.invalid/
+
+  - name: JDBC Database Connection String
+    type: jdbc_connection_string
+    regex:
+      version: 0.1
+      pattern: |
+        jdbc:[^:\x00-\x08]+:\/\/[^\/\x00-\x08]+\/[^?\x00-\x08]+\?user=[^&\x00-\x08]+&password=[^\s'"`<{$%*\x00-\x08]+
+      start: |
+        \A|\b
+      end: |
+        \z|\s|['"`<]
+    test:
+      data: |
+        jdbc:mysql://localhost:3306/mydb?user=root&password=toor
+      start_offset: 0
+      end_offset: 56

From 8583ddccddd259f5af0de0fd8d1608a9b1147135 Mon Sep 17 00:00:00 2001
From: aegilops <41705651+aegilops@users.noreply.github.com>
Date: Mon, 29 Apr 2024 17:41:21 +0100
Subject: [PATCH 28/95] Updated README.md

---
 README.md          |  2 ++
 database/README.md | 33 +++++++++++++++++++++++++++++++++
 2 files changed, 35 insertions(+)

diff --git a/README.md b/README.md
index 35ea0390..aab050c4 100644
--- a/README.md
+++ b/README.md
@@ -47,6 +47,8 @@ Custom Secret Scanning Patterns repository.
 - SQLAlchemy Database Connection String
 
 - MongoDB connection string
+
+- JDBC Database Connection String
   
 
 ### [Generic Secrets / Passwords](./generic)
diff --git a/database/README.md b/database/README.md
index 3b166a84..18da1416 100644
--- a/database/README.md
+++ b/database/README.md
@@ -379,4 +379,37 @@ Add these additional matches to the [Secret Scanning Custom Pattern](https://doc
   ^mongodb\+srv://b\*b%40f3tt%3D:%244to%40L8%3DMC@test3.test.build.10gen.cc/mydb%3F\?replicaSet=repl0
   ```
 
+</details>
+
+## JDBC Database Connection String
+
+
+
+_version: v0.1_
+
+
+
+<details>
+<summary>Pattern Format</summary>
+
+```regex
+jdbc:[^:\x00-\x08]+:\/\/[^\/\x00-\x08]+\/[^?\x00-\x08]+\?user=[^&\x00-\x08]+&password=[^\s'"`<{$%*\x00-\x08]+
+```
+
+</details>
+
+<details>
+<summary>Start Pattern</summary>
+
+```regex
+\A|\b
+```
+
+</details><details>
+<summary>End Pattern</summary>
+
+```regex
+\z|\s|['"`<]
+```
+
 </details>
\ No newline at end of file

From 6f24c7d3a2a445ac8c50ff770ecb6419f066dd29 Mon Sep 17 00:00:00 2001
From: aegilops <41705651+aegilops@users.noreply.github.com>
Date: Mon, 29 Apr 2024 18:17:52 +0100
Subject: [PATCH 29/95] Added CircleCI pattern

---
 vendors/patterns.yml | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/vendors/patterns.yml b/vendors/patterns.yml
index beada83e..bedb5a1e 100644
--- a/vendors/patterns.yml
+++ b/vendors/patterns.yml
@@ -715,3 +715,19 @@ patterns:
       - When encoded in a URL, the `+` character is replaced with `%2B`, the `/` character is replaced with `%2F`, and the `=` character is replaced with `%3D`
       - Because of the variable length of the characters (beacuse of the URL encoding), we use `{43,}` to match the signature
       - We ignore `https://files.oaiusercontent.com/` because they are URLs for images generated by ChatGPT
+
+  - name: CircleCI API token
+    type: circleci_api_token
+    description: CircleCI API token
+    regex:
+      pattern: |
+        [a-f0-9]{40}
+      start: |
+        (?i)circle[_-]?(ci[_.-]?)?(api[_.-]?)?(token|key)['"`]?(\s*[\]\)])?\s*([:,=]|[=-]>|to|[!=]={1,2}|<>)?\s*([[{])?['"`]?
+      end: |
+        ['"`\r\n,]|\z
+    test:
+      data: |
+        circle_ci_api_token: "0123456789abcdef0123456789abcdef01234567"
+      start_offset: 21
+      end_offset: 61
\ No newline at end of file

From 234408d77686f9ff87d39b9b16a1d2657be4ec34 Mon Sep 17 00:00:00 2001
From: aegilops <41705651+aegilops@users.noreply.github.com>
Date: Mon, 29 Apr 2024 18:18:03 +0100
Subject: [PATCH 30/95] Updated README.md

---
 README.md         |  2 ++
 vendors/README.md | 33 +++++++++++++++++++++++++++++++++
 2 files changed, 35 insertions(+)

diff --git a/README.md b/README.md
index aab050c4..458bdb75 100644
--- a/README.md
+++ b/README.md
@@ -198,4 +198,6 @@ Custom Secret Scanning Patterns repository.
 - Azure Access Key (legacy format)
 
 - Azure Shared Access Signature (SAS) Token
+
+- CircleCI API token
   
\ No newline at end of file
diff --git a/vendors/README.md b/vendors/README.md
index 4156a569..1e2cc9af 100644
--- a/vendors/README.md
+++ b/vendors/README.md
@@ -1485,4 +1485,37 @@ Add these additional matches to the [Secret Scanning Custom Pattern](https://doc
   ^https://files\.oaiusercontent\.com/
   ```
 
+</details>
+
+## CircleCI API token
+
+
+CircleCI API token
+_version: v0.1_
+
+
+
+<details>
+<summary>Pattern Format</summary>
+
+```regex
+[a-f0-9]{40}
+```
+
+</details>
+
+<details>
+<summary>Start Pattern</summary>
+
+```regex
+(?i)circle[_-]?(ci[_.-]?)?(api[_.-]?)?(token|key)['"`]?(\s*[\]\)])?\s*([:,=]|[=-]>|to|[!=]={1,2}|<>)?\s*([[{])?['"`]?
+```
+
+</details><details>
+<summary>End Pattern</summary>
+
+```regex
+['"`\r\n,]|\z
+```
+
 </details>
\ No newline at end of file

From 4ed2fd2ff138075b79045566765da99e924f89bf Mon Sep 17 00:00:00 2001
From: aegilops <41705651+aegilops@users.noreply.github.com>
Date: Mon, 29 Apr 2024 18:18:22 +0100
Subject: [PATCH 31/95] Fixed offsets

---
 vendors/patterns.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/vendors/patterns.yml b/vendors/patterns.yml
index bedb5a1e..06107eb5 100644
--- a/vendors/patterns.yml
+++ b/vendors/patterns.yml
@@ -729,5 +729,5 @@ patterns:
     test:
       data: |
         circle_ci_api_token: "0123456789abcdef0123456789abcdef01234567"
-      start_offset: 21
-      end_offset: 61
\ No newline at end of file
+      start_offset: 22
+      end_offset: 62

From 8e48a8ffcb6000abbbe65d2537c7e16445df1ec1 Mon Sep 17 00:00:00 2001
From: aegilops <41705651+aegilops@users.noreply.github.com>
Date: Thu, 2 May 2024 18:15:55 +0100
Subject: [PATCH 32/95] Updated PagerDuty to remove FPs

---
 vendors/patterns.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/vendors/patterns.yml b/vendors/patterns.yml
index 06107eb5..b0be7664 100644
--- a/vendors/patterns.yml
+++ b/vendors/patterns.yml
@@ -366,6 +366,9 @@ patterns:
         (\A|\b)(?i)pd_(service|api)_key['"`]?\s*([:=]|[?:]=|[=-]>|,)\s*['"`]?
       end: |
         \z|\b
+      additional_match:
+        - '[0-9]'
+        - '[A-Za-z]'
     test:
       data: pd_api_key=AAAAAAAAAAAAAAAAAAAA
       start_offset: 11

From 2ea65badb186f4425b733a2456744c35be0435b9 Mon Sep 17 00:00:00 2001
From: aegilops <41705651+aegilops@users.noreply.github.com>
Date: Thu, 2 May 2024 18:16:10 +0100
Subject: [PATCH 33/95] Updated README.md

---
 vendors/README.md | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/vendors/README.md b/vendors/README.md
index 1e2cc9af..de51c9fc 100644
--- a/vendors/README.md
+++ b/vendors/README.md
@@ -666,6 +666,27 @@ _version: v0.1_
 
 </details>
 
+<details>
+<summary>Additional Matches</summary>
+
+Add these additional matches to the [Secret Scanning Custom Pattern](https://docs.github.com/en/enterprise-cloud@latest/code-security/secret-scanning/defining-custom-patterns-for-secret-scanning#example-of-a-custom-pattern-specified-using-additional-requirements).
+
+
+
+- Match:
+
+  ```regex
+  [0-9]
+  ```
+
+- Match:
+
+  ```regex
+  [A-Za-z]
+  ```
+
+</details>
+
 ## Flickr OAuth token
 
 

From 8e604f9337ab5ebc229d4287000b376bc7607ae0 Mon Sep 17 00:00:00 2001
From: aegilops <41705651+aegilops@users.noreply.github.com>
Date: Thu, 2 May 2024 18:34:09 +0100
Subject: [PATCH 34/95] Updated YAML pattern and added new more precise YAML
 patterns

---
 configs/patterns.yml | 68 +++++++++++++++++++++++++++++++++++++++-----
 1 file changed, 61 insertions(+), 7 deletions(-)

diff --git a/configs/patterns.yml b/configs/patterns.yml
index 2d2c91ea..c2ecd0fd 100644
--- a/configs/patterns.yml
+++ b/configs/patterns.yml
@@ -21,6 +21,7 @@ patterns:
       start_offset: 18
       end_offset: 26
 
+
   - name: Hardcoded Spring SQL passwords
     description: |
       Hardcoded JDBC / Spring datasource passwords which typically are in property files or passed in at runtime
@@ -60,6 +61,7 @@ patterns:
       start_offset: 27
       end_offset: 35
 
+
   - name: Django Secret Key
     type: django_secret_key
     regex:
@@ -84,26 +86,26 @@ patterns:
   - name: YAML Static Password Fields
     type: yaml_static_password_fields
     description: |
-      Pattern to find Static passwords in YAML configuration files
+      Pattern to find hardcoded passwords in YAML configuration files
     experimental: true
     regex:
       version: 0.1
       pattern: |
-        [^\r\n'"]+
+        [^\r\n`'"]+
       start: |
-        (?:\n|\A)[ \t]*(?:secret|service_pass(wd|word|code|phrase)|pass(?:wd|word|code|phrase)?|key)[ \t]*:[ \t]*['"]?
+        (?:\n|\A)[ \t]*[a-z_-]*(?:secret|service_pass(wd|word|code|phrase)|pass(?:wd|word|code|phrase)?|key|token)[ \t]*:[ \t]*['"]?
       end: |
         ['"\r\n]|\z
       additional_not_match:
-        - ^(?:keyPassphrase|password|key|[ \t]+|\$\{[A-Za-z0-9_-]+\}|(?:str|string|int|bool)( +#.*)?),?$
+        - ^(?:keyPassphrase|password|key|[ \t]+|\$\{[^}]+}|(?:str|string|int|bool)( +#.*)?),?$
         - ^(?:.* = )?(?:None|[Tt]rue|[Ff]alse|[Nn]ull|Default(?:Type)?|Event|[A-Z]+_KEY|VERSION|NAME|update|destroy|(?:dis|en)ableEventListeners|\.\.\.),?$
         - ^(?:(?:this|self|obj)\.)(?:[A-Za-z_]+\,|[A-Za-z_].*)$
         - ^(?:[a-zA-Z_]+(?:\(\))?\.)*[a-zA-Z_]+\(\)$
         - ^\s*(?:typing\.)?(?:[Tt]uple|[Ll]ist|[Dd]ict|Callable|Iterable|Sequence|Optional|Union)\[.*$
     comments:
-      - "The hardcoded password is between 12 and 32 chars long"
-      - "Some false positives in Code might appear"
-      - "The pattern only checks for certain key words to begin the pattern (`secret`, `password`, etc.)"
+      - "The hardcoded password is any length"
+      - "Some false positives in code might appear"
+      - "The pattern only checks for certain key words to end the variable name (`secret`, `password`, etc.)"
     expected:
       - name: example.yml
         start_offset: 57
@@ -188,6 +190,7 @@ patterns:
       start_offset: 27
       end_offset: 91
 
+
   - name: .env file style secrets
     type: env_passwords
     description: |
@@ -214,3 +217,54 @@ patterns:
         SECRET_KEY=password123
       start_offset: 11
       end_offset: 22
+
+
+  - name: YAML with Base64 encoded secrets
+    type: yaml_base64
+    description: |
+      Pattern to find hardcoded Base64-encoded passwords in YAML configuration files
+    experimental: true
+    regex:
+      version: 0.1
+      pattern: |
+        (([A-Za-z+/]){4})+[A-Za-z+/]{1,2}={0,2}
+      start: |
+        (?:\n|\A)[ \t]*[a-z_-]*(?:secret|service_pass(wd|word|code|phrase)|pass(?:wd|word|code|phrase)?|key|token)[ \t]*:[ \t]*['"]?
+      end: |
+        ['"\r\n]|\z
+      additional_match:
+        - '[0-9]'
+        - '[A-Za-z]'
+        - '^.{12,}$'
+    comments:
+      - "The Base64 must contain numbers and letters and be at least 12 characters long"
+      - "Some false positives in code might appear"
+    test:
+      data: "password: AAAAAAAAAAAA00=="
+      start_offset: 10
+      end_offset: 26
+
+
+  - name: YAML with hex token
+    type: yaml_token
+    description: |
+      Pattern to find hardcoded tokens in YAML configuration files
+    experimental: true
+    regex:
+      version: 0.1
+      pattern: |
+        [0-9a-f]{32}|[0-9a-f]{40}|[0-9a-f]{64}
+      start: |
+        (?:\n|\A)[ \t]*[a-z_-]*(?:secret|service_pass(wd|word|code|phrase)|pass(?:wd|word|code|phrase)?|key|token)[ \t]*:[ \t]*['"]?
+      end: |
+        ['"\r\n]|\z
+      additional_match:
+        - '[0-9]'
+        - '[a-f]'
+    comments:
+      - "The hex token must be 32, 40 or 64 characters long, and contain numbers and letters"
+      - "Some false positives in code might appear"
+    test:
+      data: "password: 0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
+      start_offset: 10
+      end_offset: 74

From 05e0e01f5d9bbd4929e5ffe3bfd207e1417280e8 Mon Sep 17 00:00:00 2001
From: aegilops <41705651+aegilops@users.noreply.github.com>
Date: Thu, 2 May 2024 18:34:30 +0100
Subject: [PATCH 35/95] Updated README.md

---
 configs/README.md | 142 +++++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 135 insertions(+), 7 deletions(-)

diff --git a/configs/README.md b/configs/README.md
index 6557df1d..cc50e705 100644
--- a/configs/README.md
+++ b/configs/README.md
@@ -118,25 +118,25 @@ _version: v0.1_
 ## YAML Static Password Fields
 
 **⚠️ WARNING: THIS RULE IS EXPERIMENTAL AND MIGHT CAUSE A HIGH FALSE POSITIVE RATE (test before commiting to org level) ⚠️**
-Pattern to find Static passwords in YAML configuration files
+Pattern to find hardcoded passwords in YAML configuration files
 
 _version: v0.1_
 
 **Comments / Notes:**
 
 
-- The hardcoded password is between 12 and 32 chars long
+- The hardcoded password is any length
 
-- Some false positives in Code might appear
+- Some false positives in code might appear
 
-- The pattern only checks for certain key words to begin the pattern (`secret`, `password`, etc.)
+- The pattern only checks for certain key words to end the variable name (`secret`, `password`, etc.)
   
 
 <details>
 <summary>Pattern Format</summary>
 
 ```regex
-[^\r\n'"]+
+[^\r\n`'"]+
 ```
 
 </details>
@@ -145,7 +145,7 @@ _version: v0.1_
 <summary>Start Pattern</summary>
 
 ```regex
-(?:\n|\A)[ \t]*(?:secret|service_pass(wd|word|code|phrase)|pass(?:wd|word|code|phrase)?|key)[ \t]*:[ \t]*['"]?
+(?:\n|\A)[ \t]*[a-z_-]*(?:secret|service_pass(wd|word|code|phrase)|pass(?:wd|word|code|phrase)?|key|token)[ \t]*:[ \t]*['"]?
 ```
 
 </details><details>
@@ -166,7 +166,7 @@ Add these additional matches to the [Secret Scanning Custom Pattern](https://doc
 - Not Match:
 
   ```regex
-  ^(?:keyPassphrase|password|key|[ \t]+|\$\{[A-Za-z0-9_-]+\}|(?:str|string|int|bool)( +#.*)?),?$
+  ^(?:keyPassphrase|password|key|[ \t]+|\$\{[^}]+}|(?:str|string|int|bool)( +#.*)?),?$
   ```
 - Not Match:
 
@@ -383,4 +383,132 @@ Add these additional matches to the [Secret Scanning Custom Pattern](https://doc
   ^<[^>]+>$
   ```
 
+</details>
+
+## YAML with Base64 encoded secrets
+
+**⚠️ WARNING: THIS RULE IS EXPERIMENTAL AND MIGHT CAUSE A HIGH FALSE POSITIVE RATE (test before commiting to org level) ⚠️**
+Pattern to find hardcoded Base64-encoded passwords in YAML configuration files
+
+_version: v0.1_
+
+**Comments / Notes:**
+
+
+- The Base64 must contain numbers and letters and be at least 12 characters long
+
+- Some false positives in code might appear
+  
+
+<details>
+<summary>Pattern Format</summary>
+
+```regex
+(([A-Za-z+/]){4})+[A-Za-z+/]{1,2}={0,2}
+```
+
+</details>
+
+<details>
+<summary>Start Pattern</summary>
+
+```regex
+(?:\n|\A)[ \t]*[a-z_-]*(?:secret|service_pass(wd|word|code|phrase)|pass(?:wd|word|code|phrase)?|key|token)[ \t]*:[ \t]*['"]?
+```
+
+</details><details>
+<summary>End Pattern</summary>
+
+```regex
+['"\r\n]|\z
+```
+
+</details>
+
+<details>
+<summary>Additional Matches</summary>
+
+Add these additional matches to the [Secret Scanning Custom Pattern](https://docs.github.com/en/enterprise-cloud@latest/code-security/secret-scanning/defining-custom-patterns-for-secret-scanning#example-of-a-custom-pattern-specified-using-additional-requirements).
+
+
+
+- Match:
+
+  ```regex
+  [0-9]
+  ```
+
+- Match:
+
+  ```regex
+  [A-Za-z]
+  ```
+
+- Match:
+
+  ```regex
+  ^.{12,}$
+  ```
+
+</details>
+
+## YAML with hex token
+
+**⚠️ WARNING: THIS RULE IS EXPERIMENTAL AND MIGHT CAUSE A HIGH FALSE POSITIVE RATE (test before commiting to org level) ⚠️**
+Pattern to find hardcoded tokens in YAML configuration files
+
+_version: v0.1_
+
+**Comments / Notes:**
+
+
+- The hex token must be 32, 40 or 64 characters long, and contain numbers and letters
+
+- Some false positives in code might appear
+  
+
+<details>
+<summary>Pattern Format</summary>
+
+```regex
+[0-9a-f]{32}|[0-9a-f]{40}|[0-9a-f]{64}
+```
+
+</details>
+
+<details>
+<summary>Start Pattern</summary>
+
+```regex
+(?:\n|\A)[ \t]*[a-z_-]*(?:secret|service_pass(wd|word|code|phrase)|pass(?:wd|word|code|phrase)?|key|token)[ \t]*:[ \t]*['"]?
+```
+
+</details><details>
+<summary>End Pattern</summary>
+
+```regex
+['"\r\n]|\z
+```
+
+</details>
+
+<details>
+<summary>Additional Matches</summary>
+
+Add these additional matches to the [Secret Scanning Custom Pattern](https://docs.github.com/en/enterprise-cloud@latest/code-security/secret-scanning/defining-custom-patterns-for-secret-scanning#example-of-a-custom-pattern-specified-using-additional-requirements).
+
+
+
+- Match:
+
+  ```regex
+  [0-9]
+  ```
+
+- Match:
+
+  ```regex
+  [a-f]
+  ```
+
 </details>
\ No newline at end of file

From d7ced7de7f2941a6fa173d4d110acde8333f1f04 Mon Sep 17 00:00:00 2001
From: aegilops <41705651+aegilops@users.noreply.github.com>
Date: Fri, 3 May 2024 15:58:57 +0100
Subject: [PATCH 36/95] Updated PagerDuty key. Better FP handling and Token
 token= prefix to catch more results

---
 vendors/patterns.yml | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/vendors/patterns.yml b/vendors/patterns.yml
index b0be7664..fba6c959 100644
--- a/vendors/patterns.yml
+++ b/vendors/patterns.yml
@@ -363,12 +363,14 @@ patterns:
       pattern: |
         [A-Za-z0-9_-]{20}
       start: |
-        (\A|\b)(?i)pd_(service|api)_key['"`]?\s*([:=]|[?:]=|[=-]>|,)\s*['"`]?
+        (\A|\b)(?i)((pd|pagerduty)_(service|api)_key|Token token=)['"`]?\s*([:=]|[?:]=|[=-]>|,)\s*['"`]?
       end: |
         \z|\b
       additional_match:
-        - '[0-9]'
-        - '[A-Za-z]'
+        - '[A-Z]'
+        - '[a-z]'
+      additional_not_match:
+        - ^(pagerduty|pd)_(service|api)_
     test:
       data: pd_api_key=AAAAAAAAAAAAAAAAAAAA
       start_offset: 11

From 9c9a7436fc635b02aa59969e027e1f825dddbaec Mon Sep 17 00:00:00 2001
From: aegilops <41705651+aegilops@users.noreply.github.com>
Date: Fri, 3 May 2024 15:59:08 +0100
Subject: [PATCH 37/95] Updated README.md

---
 configs/README.md | 10 ++++++++--
 vendors/README.md | 17 ++++++++++++++---
 2 files changed, 22 insertions(+), 5 deletions(-)

diff --git a/configs/README.md b/configs/README.md
index cc50e705..a56bb0ac 100644
--- a/configs/README.md
+++ b/configs/README.md
@@ -395,7 +395,7 @@ _version: v0.1_
 **Comments / Notes:**
 
 
-- The Base64 must contain numbers and letters and be at least 12 characters long
+- The Base64 must contain numbers, upper case and lower case and be at least 12 characters long
 
 - Some false positives in code might appear
   
@@ -441,7 +441,13 @@ Add these additional matches to the [Secret Scanning Custom Pattern](https://doc
 - Match:
 
   ```regex
-  [A-Za-z]
+  [A-Z]
+  ```
+
+- Match:
+
+  ```regex
+  [a-z]
   ```
 
 - Match:
diff --git a/vendors/README.md b/vendors/README.md
index de51c9fc..14a91ea6 100644
--- a/vendors/README.md
+++ b/vendors/README.md
@@ -639,7 +639,13 @@ _version: v0.1_
 
 _version: v0.1_
 
+**Comments / Notes:**
+
 
+- Looks for surrounding context to confirm this is a PagerDuty API key, not some other 20-byte alphanumeric string
+
+- The `Token token=` prefix is used in an Authorization header; it's possible that a different vendor could use a similar key and this same prefix, causing results that are a different vendor's key
+  
 
 <details>
 <summary>Pattern Format</summary>
@@ -654,7 +660,7 @@ _version: v0.1_
 <summary>Start Pattern</summary>
 
 ```regex
-(\A|\b)(?i)pd_(service|api)_key['"`]?\s*([:=]|[?:]=|[=-]>|,)\s*['"`]?
+(\A|\b)(?i)((pd|pagerduty)_(service|api)_key|Token token=)['"`]?\s*([:=]|[?:]=|[=-]>|,)\s*['"`]?
 ```
 
 </details><details>
@@ -676,13 +682,18 @@ Add these additional matches to the [Secret Scanning Custom Pattern](https://doc
 - Match:
 
   ```regex
-  [0-9]
+  [A-Z]
   ```
 
 - Match:
 
   ```regex
-  [A-Za-z]
+  [a-z]
+  ```
+- Not Match:
+
+  ```regex
+  ^(pagerduty|pd)_(service|api)_
   ```
 
 </details>

From 2291e09497eca2a12b7e49c0786bb0b502228274 Mon Sep 17 00:00:00 2001
From: aegilops <41705651+aegilops@users.noreply.github.com>
Date: Fri, 3 May 2024 16:13:50 +0100
Subject: [PATCH 38/95] Updated PagerDuty key. Fixed Token token= version

---
 vendors/patterns.yml | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/vendors/patterns.yml b/vendors/patterns.yml
index fba6c959..8816ed04 100644
--- a/vendors/patterns.yml
+++ b/vendors/patterns.yml
@@ -363,7 +363,7 @@ patterns:
       pattern: |
         [A-Za-z0-9_-]{20}
       start: |
-        (\A|\b)(?i)((pd|pagerduty)_(service|api)_key|Token token=)['"`]?\s*([:=]|[?:]=|[=-]>|,)\s*['"`]?
+        (\A|\b)(?i)((pd|pagerduty)_(service|api)_key['"`]?\s*([:=]|[?:]=|[=-]>|,)\s*['"`]?|Token token=)
       end: |
         \z|\b
       additional_match:
@@ -374,6 +374,9 @@ patterns:
     test:
       data: pd_api_key=AAAAAAAAAAAAAAAAAAAA
       start_offset: 11
+    comments:
+      - "Looks for surrounding context to confirm this is a PagerDuty API key, not some other 20-byte alphanumeric string"
+      - "The `Token token=` prefix is used in an Authorization header; it's possible that a different vendor could use a similar key and this same prefix, causing results that are a different vendor's key"
 
   - name: Flickr OAuth token
     type: flickr_oauth_token

From f73a41bf309e24a69e94d518d5ae8a6b199c2fee Mon Sep 17 00:00:00 2001
From: aegilops <41705651+aegilops@users.noreply.github.com>
Date: Fri, 3 May 2024 16:13:58 +0100
Subject: [PATCH 39/95] Updated README.md

---
 vendors/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/vendors/README.md b/vendors/README.md
index 14a91ea6..953ea734 100644
--- a/vendors/README.md
+++ b/vendors/README.md
@@ -660,7 +660,7 @@ _version: v0.1_
 <summary>Start Pattern</summary>
 
 ```regex
-(\A|\b)(?i)((pd|pagerduty)_(service|api)_key|Token token=)['"`]?\s*([:=]|[?:]=|[=-]>|,)\s*['"`]?
+(\A|\b)(?i)((pd|pagerduty)_(service|api)_key['"`]?\s*([:=]|[?:]=|[=-]>|,)\s*['"`]?|Token token=)
 ```
 
 </details><details>

From 9d6e93ea843c5c5334152147bd90363f7cb150d8 Mon Sep 17 00:00:00 2001
From: aegilops <41705651+aegilops@users.noreply.github.com>
Date: Fri, 3 May 2024 18:14:41 +0100
Subject: [PATCH 40/95] Updated YAML pattern and added new more-specific YAML
 patterns to reduce FPs

---
 configs/patterns.yml | 185 ++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 175 insertions(+), 10 deletions(-)

diff --git a/configs/patterns.yml b/configs/patterns.yml
index c2ecd0fd..c89aab09 100644
--- a/configs/patterns.yml
+++ b/configs/patterns.yml
@@ -91,21 +91,22 @@ patterns:
     regex:
       version: 0.1
       pattern: |
-        [^\r\n`'"]+
+        [^\r\n`'"\x00-\x08]+
       start: |
-        (?:\n|\A)[ \t]*[a-z_-]*(?:secret|service_pass(wd|word|code|phrase)|pass(?:wd|word|code|phrase)?|key|token)[ \t]*:[ \t]*['"]?
+        (?:\n|\A)[ \t]*(?i)[a-z_-]*(?:secret|service_pass(wd|word|code|phrase)|pass(?:wd|word|code|phrase)?|key|token)[ \t]*:[ \t]*['"]?
       end: |
         ['"\r\n]|\z
       additional_not_match:
         - ^(?:keyPassphrase|password|key|[ \t]+|\$\{[^}]+}|(?:str|string|int|bool)( +#.*)?),?$
         - ^(?:.* = )?(?:None|[Tt]rue|[Ff]alse|[Nn]ull|Default(?:Type)?|Event|[A-Z]+_KEY|VERSION|NAME|update|destroy|(?:dis|en)ableEventListeners|\.\.\.),?$
         - ^(?:(?:this|self|obj)\.)(?:[A-Za-z_]+\,|[A-Za-z_].*)$
-        - ^(?:[a-zA-Z_]+(?:\(\))?\.)*[a-zA-Z_]+\(\)$
+        - ^(?:(?:[a-zA-Z_]+(?:\(\))?\.)*[a-zA-Z_]+\(\)|\|\s*)$
         - ^\s*(?:typing\.)?(?:[Tt]uple|[Ll]ist|[Dd]ict|Callable|Iterable|Sequence|Optional|Union)\[.*$
     comments:
       - "The hardcoded password is any length"
       - "Some false positives in code might appear"
       - "The pattern only checks for certain key words to end the variable name (`secret`, `password`, etc.)"
+      - "Does not allow for multline blocks"
     expected:
       - name: example.yml
         start_offset: 57
@@ -118,6 +119,114 @@ patterns:
       start_offset: 10
       end_offset: 21
 
+
+  - name: YAML hardcoded passwords (plain scalars)
+    type: yaml_passwords_plain
+    description: |
+      Hardcoded passwords in YAML configuration files, using plain scalars
+    experimental: true
+    regex:
+      version: 0.1
+      pattern: |
+        [^\r\n`'"\x00-\x08]+
+      start: |
+        (?:\n|\A)[ \t]*(?i)[a-z_-]*(?:secret|service_pass(wd|word|code|phrase)|pass(?:wd|word|code|phrase)?|key|token)[ \t]*:[ \t]*
+      end: |
+        [\r\n]|\z
+      additional_not_match:
+        - ^(?:keyPassphrase|password|key|[ \t]+|\$\{[^}]+}|(?:str|string|int|bool)( +#.*)?),?$
+        - ^(?:.* = )?(?:None|[Tt]rue|[Ff]alse|[Nn]ull|Default(?:Type)?|Event|[A-Z]+_KEY|VERSION|NAME|update|destroy|(?:dis|en)ableEventListeners|\.\.\.),?$
+        - '^(?:(?:(?:this|self|obj)\.)(?:[A-Za-z_]+\,|[A-Za-z_].*)|([!&*{}[\],#|>@`"''%]|[:?-] ).*)$'
+        - ^(?:[a-zA-Z_]+(?:\(\))?\.)*[a-zA-Z_]+\(\)$
+        - ^\s*(?:typing\.)?(?:[Tt]uple|[Ll]ist|[Dd]ict|Callable|Iterable|Sequence|Optional|Union)\[.*$
+    comments:
+      - "The hardcoded password is any length"
+      - "Some false positives in code might appear"
+      - "The pattern only checks for certain key words to end the variable name (`secret`, `password`, etc.)"
+      - "Only allows for plain scalars, not quoted or multi-line, to better control false positives"
+    test:
+      data: "password: password123"
+      start_offset: 10
+      end_offset: 21
+
+
+  - name: YAML hardcoded passwords (single quoted strings)
+    type: yaml_passwords_single_quoted
+    description: |
+      Hardcoded passwords in YAML configuration files, using single quotes
+    experimental: true
+    regex:
+      version: 0.1
+      pattern: |
+        [^\r\n'\x00-\x08]+
+      start: |
+        (?:\n|\A)[ \t]*(?i)[a-z_-]*(?:secret|service_pass(wd|word|code|phrase)|pass(?:wd|word|code|phrase)?|key|token)[ \t]*:[ \t]*'
+      end: |
+        '([ \t]*[\r\n]|\z)
+    comments:
+      - "The hardcoded password is any length"
+      - "Some false positives in code might appear"
+      - "The pattern only checks for certain key words to end the variable name (`secret`, `password`, etc.)"
+      - "Only allows for only single-quoted passwords, to better control false positives"
+    test:
+      data: "password: 'password123'"
+      start_offset: 10
+      end_offset: 21
+
+
+  - name: YAML hardcoded passwords (double quoted strings)
+    type: yaml_passwords_single_quoted
+    description: |
+      Hardcoded passwords in YAML configuration files, using single quotes
+    experimental: true
+    regex:
+      version: 0.1
+      pattern: |
+        [^\r\n"\x00-\x08]+
+      start: |
+        (?:\n|\A)[ \t]*(?i)[a-z_-]*(?:secret|service_pass(wd|word|code|phrase)|pass(?:wd|word|code|phrase)?|key|token)[ \t]*:[ \t]*"
+      end: |
+        "([ \t]*[\r\n]|\z)
+    comments:
+      - "The hardcoded password is any length"
+      - "Some false positives in code might appear"
+      - "The pattern only checks for certain key words to end the variable name (`secret`, `password`, etc.)"
+      - "Only allows for only double-quoted passwords, to better control false positives"
+    test:
+      data: |
+        password: "password123"
+
+      start_offset: 10
+      end_offset: 21
+
+
+  - name: YAML hardcoded passwords (multiline strings)
+    type: yaml_passwords_multiline
+    description: |
+      Hardcoded passwords in YAML configuration files, using multiline strings
+    experimental: true
+    regex:
+      version: 0.1
+      pattern: |
+        [^\x00-\x08]+?
+      start: |
+        (?:\n|\A)[ \t]*(?i)[a-z_-]*(?:secret|service_pass(wd|word|code|phrase)|pass(?:wd|word|code|phrase)?|key|token)[ \t]*:[ \t]*[|>][+-]?[ \t]*\n[ \t]+
+      end: |
+        \n\n|\r\n\r\n|(\n|\r\n)[ \t]+\S+:|(\n|\r\n)\.\.\.[ \t\n\r]|\z
+    comments:
+      - "The hardcoded password is any length"
+      - "Some false positives in code might appear"
+      - "The pattern only checks for certain key words to end the variable name (`secret`, `password`, etc.)"
+      - "This will catch the start of a multiline password, but the end will not be found if it is on a different line"
+    test:
+      data: |
+        password: |
+          password123
+          andsomemore
+        other_field: foo
+      # TODO: add start_offset and end_offset
+
+
   - name: GitHub Actions SHA Checker
     type: github_actions_sha_checker
     comments:
@@ -222,22 +331,23 @@ patterns:
   - name: YAML with Base64 encoded secrets
     type: yaml_base64
     description: |
-      Pattern to find hardcoded Base64-encoded passwords in YAML configuration files
+      Hardcoded Base64-encoded passwords in YAML configuration files
     experimental: true
     regex:
       version: 0.1
       pattern: |
-        (([A-Za-z+/]){4})+[A-Za-z+/]{1,2}={0,2}
+        (([A-Za-z0-9+/]){4})+[A-Za-z0-9+/]{1,2}={0,2}
       start: |
-        (?:\n|\A)[ \t]*[a-z_-]*(?:secret|service_pass(wd|word|code|phrase)|pass(?:wd|word|code|phrase)?|key|token)[ \t]*:[ \t]*['"]?
+        (?:\n|\A)[ \t]*(?i)[a-z_-]*(?:secret|service_pass(wd|word|code|phrase)|pass(?:wd|word|code|phrase)?|key|token)[ \t]*:[ \t]*(['"]?|[|>]-?[ \t]*\n[ \t]*)
       end: |
         ['"\r\n]|\z
       additional_match:
         - '[0-9]'
-        - '[A-Za-z]'
+        - '[A-Z]'
+        - '[a-z]'
         - '^.{12,}$'
     comments:
-      - "The Base64 must contain numbers and letters and be at least 12 characters long"
+      - "The Base64 must contain numbers, upper case and lower case and be at least 12 characters long"
       - "Some false positives in code might appear"
     test:
       data: "password: AAAAAAAAAAAA00=="
@@ -248,14 +358,14 @@ patterns:
   - name: YAML with hex token
     type: yaml_token
     description: |
-      Pattern to find hardcoded tokens in YAML configuration files
+      Hardcoded hex-encoded tokens in YAML configuration files
     experimental: true
     regex:
       version: 0.1
       pattern: |
         [0-9a-f]{32}|[0-9a-f]{40}|[0-9a-f]{64}
       start: |
-        (?:\n|\A)[ \t]*[a-z_-]*(?:secret|service_pass(wd|word|code|phrase)|pass(?:wd|word|code|phrase)?|key|token)[ \t]*:[ \t]*['"]?
+        (?:\n|\A)[ \t]*(?i)[a-z_-]*(?:secret|service_pass(wd|word|code|phrase)|pass(?:wd|word|code|phrase)?|key|token)[ \t]*:[ \t]*(['"]?|[|>]-?[ \t]*\n[ \t]*)
       end: |
         ['"\r\n]|\z
       additional_match:
@@ -268,3 +378,58 @@ patterns:
       data: "password: 0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
       start_offset: 10
       end_offset: 74
+
+
+  - name: JSON with Base64 encoded secrets
+    type: json_base64
+    description: |
+      Hardcoded Base64-encoded passwords in JSON configuration files
+    experimental: true
+    regex:
+      version: 0.1
+      pattern: |
+        (([A-Za-z0-9+/]){4})+[A-Za-z0-9+/]{1,2}={0,2}
+      start: |
+        [{[,][ \t]*[ \t\r\n]*"(?i)[a-z_-]*(?:secret|service_pass(wd|word|code|phrase)|pass(?:wd|word|code|phrase)?|key|token)"[ \t]*:[ \t]*"
+      end: |
+        "[ \t\r\n]*[,}\]]
+      additional_match:
+        - '[0-9]'
+        - '[A-Z]'
+        - '[a-z]'
+        - '^.{12,}$'
+    comments:
+      - "The Base64 must contain numbers, upper case and lower case and be at least 12 characters long"
+      - "This may match in code, such as Python, that resembles JSON"
+      - "This will not match some isolated fragments of JSON, so be aware of that when testing it"
+    test:
+      data: |
+        "password": "AAAAAAAAAAAA00==",
+      start_offset: 13
+      end_offset: 29
+
+
+  - name: JSON with hex encoded secrets
+    type: json_token
+    description: |
+      Hardcoded hex-encoded tokens in JSON configuration files
+    experimental: true
+    regex:
+      version: 0.1
+      pattern: |
+        [0-9a-f]{32}|[0-9a-f]{40}|[0-9a-f]{64}
+      start: |
+        [{[,][ \t]*[ \t\r\n]*"(?i)[a-z_-]*(?:secret|service_pass(wd|word|code|phrase)|pass(?:wd|word|code|phrase)?|key|token)"[ \t]*:[ \t]*"
+      end: |
+        "[ \t\r\n]*[,}\]]
+      additional_match:
+        - '[0-9]'
+        - '[a-f]'
+    comments:
+      - "The hex token must be 32, 40 or 64 characters long, and contain numbers and letters"
+      - "Some false positives in code might appear"
+    test:
+      data: |
+        {"password": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", "user": "admin"}
+      start_offset: 14
+      end_offset: 78

From ad5726df58e2a0ad7e1359c8c7f3760118c6fb1a Mon Sep 17 00:00:00 2001
From: aegilops <41705651+aegilops@users.noreply.github.com>
Date: Fri, 3 May 2024 18:14:51 +0100
Subject: [PATCH 41/95] Updated README.md

---
 configs/README.md | 362 +++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 355 insertions(+), 7 deletions(-)

diff --git a/configs/README.md b/configs/README.md
index a56bb0ac..26744eb7 100644
--- a/configs/README.md
+++ b/configs/README.md
@@ -130,13 +130,15 @@ _version: v0.1_
 - Some false positives in code might appear
 
 - The pattern only checks for certain key words to end the variable name (`secret`, `password`, etc.)
+
+- Does not allow for multline blocks
   
 
 <details>
 <summary>Pattern Format</summary>
 
 ```regex
-[^\r\n`'"]+
+[^\r\n`'"\x00-\x08]+
 ```
 
 </details>
@@ -145,7 +147,7 @@ _version: v0.1_
 <summary>Start Pattern</summary>
 
 ```regex
-(?:\n|\A)[ \t]*[a-z_-]*(?:secret|service_pass(wd|word|code|phrase)|pass(?:wd|word|code|phrase)?|key|token)[ \t]*:[ \t]*['"]?
+(?:\n|\A)[ \t]*(?i)[a-z_-]*(?:secret|service_pass(wd|word|code|phrase)|pass(?:wd|word|code|phrase)?|key|token)[ \t]*:[ \t]*['"]?
 ```
 
 </details><details>
@@ -180,6 +182,84 @@ Add these additional matches to the [Secret Scanning Custom Pattern](https://doc
   ```
 - Not Match:
 
+  ```regex
+  ^(?:(?:[a-zA-Z_]+(?:\(\))?\.)*[a-zA-Z_]+\(\)|\|\s*)$
+  ```
+- Not Match:
+
+  ```regex
+  ^\s*(?:typing\.)?(?:[Tt]uple|[Ll]ist|[Dd]ict|Callable|Iterable|Sequence|Optional|Union)\[.*$
+  ```
+
+</details>
+
+## YAML hardcoded passwords (plain scalars)
+
+**⚠️ WARNING: THIS RULE IS EXPERIMENTAL AND MIGHT CAUSE A HIGH FALSE POSITIVE RATE (test before commiting to org level) ⚠️**
+Hardcoded passwords in YAML configuration files, using plain scalars
+
+_version: v0.1_
+
+**Comments / Notes:**
+
+
+- The hardcoded password is any length
+
+- Some false positives in code might appear
+
+- The pattern only checks for certain key words to end the variable name (`secret`, `password`, etc.)
+
+- Only allows for plain scalars, not quoted or multi-line, to better control false positives
+  
+
+<details>
+<summary>Pattern Format</summary>
+
+```regex
+[^\r\n`'"\x00-\x08]+
+```
+
+</details>
+
+<details>
+<summary>Start Pattern</summary>
+
+```regex
+(?:\n|\A)[ \t]*(?i)[a-z_-]*(?:secret|service_pass(wd|word|code|phrase)|pass(?:wd|word|code|phrase)?|key|token)[ \t]*:[ \t]*
+```
+
+</details><details>
+<summary>End Pattern</summary>
+
+```regex
+[\r\n]|\z
+```
+
+</details>
+
+<details>
+<summary>Additional Matches</summary>
+
+Add these additional matches to the [Secret Scanning Custom Pattern](https://docs.github.com/en/enterprise-cloud@latest/code-security/secret-scanning/defining-custom-patterns-for-secret-scanning#example-of-a-custom-pattern-specified-using-additional-requirements).
+
+
+- Not Match:
+
+  ```regex
+  ^(?:keyPassphrase|password|key|[ \t]+|\$\{[^}]+}|(?:str|string|int|bool)( +#.*)?),?$
+  ```
+- Not Match:
+
+  ```regex
+  ^(?:.* = )?(?:None|[Tt]rue|[Ff]alse|[Nn]ull|Default(?:Type)?|Event|[A-Z]+_KEY|VERSION|NAME|update|destroy|(?:dis|en)ableEventListeners|\.\.\.),?$
+  ```
+- Not Match:
+
+  ```regex
+  ^(?:(?:(?:this|self|obj)\.)(?:[A-Za-z_]+\,|[A-Za-z_].*)|([!&*{}[\],#|>@`"'%]|[:?-] ).*)$
+  ```
+- Not Match:
+
   ```regex
   ^(?:[a-zA-Z_]+(?:\(\))?\.)*[a-zA-Z_]+\(\)$
   ```
@@ -191,6 +271,138 @@ Add these additional matches to the [Secret Scanning Custom Pattern](https://doc
 
 </details>
 
+## YAML hardcoded passwords (single quoted strings)
+
+**⚠️ WARNING: THIS RULE IS EXPERIMENTAL AND MIGHT CAUSE A HIGH FALSE POSITIVE RATE (test before commiting to org level) ⚠️**
+Hardcoded passwords in YAML configuration files, using single quotes
+
+_version: v0.1_
+
+**Comments / Notes:**
+
+
+- The hardcoded password is any length
+
+- Some false positives in code might appear
+
+- The pattern only checks for certain key words to end the variable name (`secret`, `password`, etc.)
+
+- Only allows for only single-quoted passwords, to better control false positives
+  
+
+<details>
+<summary>Pattern Format</summary>
+
+```regex
+[^\r\n'\x00-\x08]+
+```
+
+</details>
+
+<details>
+<summary>Start Pattern</summary>
+
+```regex
+(?:\n|\A)[ \t]*(?i)[a-z_-]*(?:secret|service_pass(wd|word|code|phrase)|pass(?:wd|word|code|phrase)?|key|token)[ \t]*:[ \t]*'
+```
+
+</details><details>
+<summary>End Pattern</summary>
+
+```regex
+'([ \t]*[\r\n]|\z)
+```
+
+</details>
+
+## YAML hardcoded passwords (double quoted strings)
+
+**⚠️ WARNING: THIS RULE IS EXPERIMENTAL AND MIGHT CAUSE A HIGH FALSE POSITIVE RATE (test before commiting to org level) ⚠️**
+Hardcoded passwords in YAML configuration files, using single quotes
+
+_version: v0.1_
+
+**Comments / Notes:**
+
+
+- The hardcoded password is any length
+
+- Some false positives in code might appear
+
+- The pattern only checks for certain key words to end the variable name (`secret`, `password`, etc.)
+
+- Only allows for only double-quoted passwords, to better control false positives
+  
+
+<details>
+<summary>Pattern Format</summary>
+
+```regex
+[^\r\n"\x00-\x08]+
+```
+
+</details>
+
+<details>
+<summary>Start Pattern</summary>
+
+```regex
+(?:\n|\A)[ \t]*(?i)[a-z_-]*(?:secret|service_pass(wd|word|code|phrase)|pass(?:wd|word|code|phrase)?|key|token)[ \t]*:[ \t]*"
+```
+
+</details><details>
+<summary>End Pattern</summary>
+
+```regex
+"([ \t]*[\r\n]|\z)
+```
+
+</details>
+
+## YAML hardcoded passwords (multiline strings)
+
+**⚠️ WARNING: THIS RULE IS EXPERIMENTAL AND MIGHT CAUSE A HIGH FALSE POSITIVE RATE (test before commiting to org level) ⚠️**
+Hardcoded passwords in YAML configuration files, using multiline strings
+
+_version: v0.1_
+
+**Comments / Notes:**
+
+
+- The hardcoded password is any length
+
+- Some false positives in code might appear
+
+- The pattern only checks for certain key words to end the variable name (`secret`, `password`, etc.)
+
+- This will catch the start of a multiline password, but the end will not be found if it is on a different line
+  
+
+<details>
+<summary>Pattern Format</summary>
+
+```regex
+[^\x00-\x08]+?
+```
+
+</details>
+
+<details>
+<summary>Start Pattern</summary>
+
+```regex
+(?:\n|\A)[ \t]*(?i)[a-z_-]*(?:secret|service_pass(wd|word|code|phrase)|pass(?:wd|word|code|phrase)?|key|token)[ \t]*:[ \t]*[|>][+-]?[ \t]*\n[ \t]+
+```
+
+</details><details>
+<summary>End Pattern</summary>
+
+```regex
+\n\n|\r\n\r\n|(\n|\r\n)[ \t]+\S+:|(\n|\r\n)\.\.\.[ \t\n\r]|\z
+```
+
+</details>
+
 ## GitHub Actions SHA Checker
 
 
@@ -388,7 +600,7 @@ Add these additional matches to the [Secret Scanning Custom Pattern](https://doc
 ## YAML with Base64 encoded secrets
 
 **⚠️ WARNING: THIS RULE IS EXPERIMENTAL AND MIGHT CAUSE A HIGH FALSE POSITIVE RATE (test before commiting to org level) ⚠️**
-Pattern to find hardcoded Base64-encoded passwords in YAML configuration files
+Hardcoded Base64-encoded passwords in YAML configuration files
 
 _version: v0.1_
 
@@ -404,7 +616,7 @@ _version: v0.1_
 <summary>Pattern Format</summary>
 
 ```regex
-(([A-Za-z+/]){4})+[A-Za-z+/]{1,2}={0,2}
+(([A-Za-z0-9+/]){4})+[A-Za-z0-9+/]{1,2}={0,2}
 ```
 
 </details>
@@ -413,7 +625,7 @@ _version: v0.1_
 <summary>Start Pattern</summary>
 
 ```regex
-(?:\n|\A)[ \t]*[a-z_-]*(?:secret|service_pass(wd|word|code|phrase)|pass(?:wd|word|code|phrase)?|key|token)[ \t]*:[ \t]*['"]?
+(?:\n|\A)[ \t]*(?i)[a-z_-]*(?:secret|service_pass(wd|word|code|phrase)|pass(?:wd|word|code|phrase)?|key|token)[ \t]*:[ \t]*(['"]?|[|>]-?[ \t]*\n[ \t]*)
 ```
 
 </details><details>
@@ -461,7 +673,7 @@ Add these additional matches to the [Secret Scanning Custom Pattern](https://doc
 ## YAML with hex token
 
 **⚠️ WARNING: THIS RULE IS EXPERIMENTAL AND MIGHT CAUSE A HIGH FALSE POSITIVE RATE (test before commiting to org level) ⚠️**
-Pattern to find hardcoded tokens in YAML configuration files
+Hardcoded hex-encoded tokens in YAML configuration files
 
 _version: v0.1_
 
@@ -486,7 +698,7 @@ _version: v0.1_
 <summary>Start Pattern</summary>
 
 ```regex
-(?:\n|\A)[ \t]*[a-z_-]*(?:secret|service_pass(wd|word|code|phrase)|pass(?:wd|word|code|phrase)?|key|token)[ \t]*:[ \t]*['"]?
+(?:\n|\A)[ \t]*(?i)[a-z_-]*(?:secret|service_pass(wd|word|code|phrase)|pass(?:wd|word|code|phrase)?|key|token)[ \t]*:[ \t]*(['"]?|[|>]-?[ \t]*\n[ \t]*)
 ```
 
 </details><details>
@@ -505,6 +717,142 @@ Add these additional matches to the [Secret Scanning Custom Pattern](https://doc
 
 
 
+- Match:
+
+  ```regex
+  [0-9]
+  ```
+
+- Match:
+
+  ```regex
+  [a-f]
+  ```
+
+</details>
+
+## JSON with Base64 encoded secrets
+
+**⚠️ WARNING: THIS RULE IS EXPERIMENTAL AND MIGHT CAUSE A HIGH FALSE POSITIVE RATE (test before commiting to org level) ⚠️**
+Hardcoded Base64-encoded passwords in JSON configuration files
+
+_version: v0.1_
+
+**Comments / Notes:**
+
+
+- The Base64 must contain numbers, upper case and lower case and be at least 12 characters long
+
+- This may match in code, such as Python, that resembles JSON
+
+- This will not match some isolated fragments of JSON, so be aware of that when testing it
+  
+
+<details>
+<summary>Pattern Format</summary>
+
+```regex
+(([A-Za-z0-9+/]){4})+[A-Za-z0-9+/]{1,2}={0,2}
+```
+
+</details>
+
+<details>
+<summary>Start Pattern</summary>
+
+```regex
+[{[,][ \t]*[ \t\r\n]*"(?i)[a-z_-]*(?:secret|service_pass(wd|word|code|phrase)|pass(?:wd|word|code|phrase)?|key|token)"[ \t]*:[ \t]*"
+```
+
+</details><details>
+<summary>End Pattern</summary>
+
+```regex
+"[ \t\r\n]*[,}\]]
+```
+
+</details>
+
+<details>
+<summary>Additional Matches</summary>
+
+Add these additional matches to the [Secret Scanning Custom Pattern](https://docs.github.com/en/enterprise-cloud@latest/code-security/secret-scanning/defining-custom-patterns-for-secret-scanning#example-of-a-custom-pattern-specified-using-additional-requirements).
+
+
+
+- Match:
+
+  ```regex
+  [0-9]
+  ```
+
+- Match:
+
+  ```regex
+  [A-Z]
+  ```
+
+- Match:
+
+  ```regex
+  [a-z]
+  ```
+
+- Match:
+
+  ```regex
+  ^.{12,}$
+  ```
+
+</details>
+
+## JSON with hex encoded secrets
+
+**⚠️ WARNING: THIS RULE IS EXPERIMENTAL AND MIGHT CAUSE A HIGH FALSE POSITIVE RATE (test before commiting to org level) ⚠️**
+Hardcoded hex-encoded tokens in JSON configuration files
+
+_version: v0.1_
+
+**Comments / Notes:**
+
+
+- The hex token must be 32, 40 or 64 characters long, and contain numbers and letters
+
+- Some false positives in code might appear
+  
+
+<details>
+<summary>Pattern Format</summary>
+
+```regex
+[0-9a-f]{32}|[0-9a-f]{40}|[0-9a-f]{64}
+```
+
+</details>
+
+<details>
+<summary>Start Pattern</summary>
+
+```regex
+[{[,][ \t]*[ \t\r\n]*"(?i)[a-z_-]*(?:secret|service_pass(wd|word|code|phrase)|pass(?:wd|word|code|phrase)?|key|token)"[ \t]*:[ \t]*"
+```
+
+</details><details>
+<summary>End Pattern</summary>
+
+```regex
+"[ \t\r\n]*[,}\]]
+```
+
+</details>
+
+<details>
+<summary>Additional Matches</summary>
+
+Add these additional matches to the [Secret Scanning Custom Pattern](https://docs.github.com/en/enterprise-cloud@latest/code-security/secret-scanning/defining-custom-patterns-for-secret-scanning#example-of-a-custom-pattern-specified-using-additional-requirements).
+
+
+
 - Match:
 
   ```regex

From 456eee7a35766cccab141ad9b8c4beb126a608a2 Mon Sep 17 00:00:00 2001
From: aegilops <41705651+aegilops@users.noreply.github.com>
Date: Fri, 3 May 2024 18:15:43 +0100
Subject: [PATCH 42/95] Updated test data for JSON pattern

---
 configs/patterns.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/configs/patterns.yml b/configs/patterns.yml
index c89aab09..8543216b 100644
--- a/configs/patterns.yml
+++ b/configs/patterns.yml
@@ -404,9 +404,9 @@ patterns:
       - "This will not match some isolated fragments of JSON, so be aware of that when testing it"
     test:
       data: |
-        "password": "AAAAAAAAAAAA00==",
-      start_offset: 13
-      end_offset: 29
+        {"password": "AAAAAAAAAAAA00==", "user": "admin"}
+      start_offset: 14
+      end_offset: 30
 
 
   - name: JSON with hex encoded secrets

From 648d60470de6ef15a8d73efd95425409f4ded517 Mon Sep 17 00:00:00 2001
From: aegilops <41705651+aegilops@users.noreply.github.com>
Date: Thu, 9 May 2024 15:21:03 +0100
Subject: [PATCH 43/95] Removed more FPs from generic passwords pattern

---
 generic/patterns.yml | 11 ++++-------
 1 file changed, 4 insertions(+), 7 deletions(-)

diff --git a/generic/patterns.yml b/generic/patterns.yml
index afe9aa3b..b114288b 100644
--- a/generic/patterns.yml
+++ b/generic/patterns.yml
@@ -4,7 +4,7 @@ patterns:
   - name: Generic Passwords
     type: generic_passwords
     regex:
-      version: 0.4
+      version: 0.5
       pattern: |
         [a-zA-Z0-9!.,$%&*+?^_`{|}()[\]\\/~-][a-zA-Z0-9\t !.,$%&*+?^_`{|}()[\]\\/~-]*
       start: |
@@ -14,23 +14,21 @@ patterns:
       additional_not_match:
         # something that means "password" or a placeholder name - either a variable or a placeholder
         # a literal value or config switch
-        - ^(?i)(?:[a-z0-9_.]*,\s*)?(?:str\()?[[<(]?(?:(?:(?:user|key)_?)?(?:[a-zA-Z0-9._]+[_.])?(?:the )?(?:pass?(wo?r?d|code|phrase)|pass|pwd|secret|token|tok|redacted|placeholder|dummy|pw|thephrase)|write|read|on|off|true|false|none|null|nil|undefined|eof|ignore|eol|git|yes|no|y|n),?[\]>)]?(?:\)\s*\{)?\\?( or )?$
+        - ^(?i)(?:[a-z0-9_.]*,\s*)?(?:str\()?[[<(]?(?:(?:(?:user|key)_?)?(?:[a-zA-Z0-9._]+[_.])?(?:the )?(?:pass?(wo?r?d|code|phrase)|pass|pwd|secret|token|tok|redacted|placeholder|dummy|pw|thephrase)|write|read|on|off|true|false|none|null( \? )?|nil|undefined|eof|ignore|eol|git|yes|no|y|n),?[\]>)]?(?:\)\s*\{)?\\?( or | \|\| )?$
         # Python type hints, Swift typing
         - ^\s*(?:(?:typing\.)?(?:(?:[Tt]uple|[Ll]ist|[Dd]ict|Callable|Iterable|Sequence|Optional|Union)\[.*|(?:int|str|float|(?:typing.)?Any|None|bytes|bool|ReadableBuffer)\s*(?:[,|].*)?|(?:Int|Swift\.Int|Int32)\.*))\s*$
         # ..., \, , \n, \0, ',' and other single chars, smilies, hex, digits, nothing at all,
         # directories, regex, format string placeholder, urllib demo passphrase, "optional" in docs, a variable substitution, or surrounded by brackets of various kinds
         # all with possible ',' and surrounding whitespace, possibly with a following comment
-        - ^\s*(?:\.\.\.|\\|\\n|\\0|[,()[\]{}`.]\\?|-[)(]|0x[A-Fa-f0-9]+|[0-9]{1,4}|(?:~|/tmp|\.\.|\.)|\\{1,2}w\+/g,( \\?)?|%[sr]|geheim\$parole|\([Oo]ptional\).*|\$?(?:\{\{?[^}]+\}\}?|\(\(?[^)]+\)\)?|\[\[?[^\]+]\]\]?))?,?\s*(?:\s*(?:/\*|#|//).*)?$
+        - ^\s*(?:\.\.\.|\\|\\n|\\0|\?|[,()[\]{}`.]\\?|-[)(]|\\f21b|0x[A-Fa-f0-9]+|[0-9]{1,4}|(?:~|/tmp|\.\.|\.)|\\{1,2}w\+/g,( \\?)?|%[sr]|geheim\$parole|\([Oo]ptional\).*|\$?(?:\{\{?[^}]+\}\}?|\(\(?[^)]+\)\)?|\[\[?[^\]+]\]\]?)|(before|hover)(,| \{))?,?\s*(?:\s*(?:/\*|#|//).*)?$
         # function definitions, e.g. Javascript, function calls or variable declaration
-        - ^(?:function\s*\([^)]*\)\s*{\s*.*|\([^)]*\)\s*=>\s*(?:{\s*|[^;)]+[;)])|(?:new )?[a-zA-Z0-9_.]+\(.*|(?:public|private) [A-Za-z0-9_]+ \{)$
+        - ^(?:function\s*\([^)]*\)\s*{\s*.*|\([^)]*\)\s*=>\s*(?:{\s*|[^;)]+[;)])|(?:new |\([A-Za-z]+\)\s*)?[a-zA-Z0-9_.]+\s*\(.*|(?:public|private) [A-Za-z0-9_]+ \{|[A-Za-z0-9_.-]+\s*\) \{)$
         # reference to a member variable, index into a variable, bash variables, perl hash key index, environment vars
         - ^\s*(?:(?:self|this)\.[a-zA-Z_][a-zA-Z0-9_.]+[,[]?|[a-zA-Z0-9_.]+\[(?:[a-zA-Z0-9_.]+)?\]?|\$(?:[1-9]|[A-Za-z0-9_]+)\{?|os\.environ\[[^\]]\]|process\.env\.[A-Z0-9_]+)\s*(?:,|\|\||&&)?\s*$
-
     test:
       data: password=Password123
       start_offset: 9
       end_offset: -1
-
     expected:
       - name: passwords.js
         start_offset: 14
@@ -41,7 +39,6 @@ patterns:
       - name: passwords.js
         start_offset: 97
         end_offset: 112
-
     comments:
       - "`password`, `secret`, `key`, or password like prefix (fuzzy)"
       - "Delimiters like `=` or `:` (with padding)"

From 96a6c793954a3c2af8979a4bca5ea911e23d7e5e Mon Sep 17 00:00:00 2001
From: aegilops <41705651+aegilops@users.noreply.github.com>
Date: Thu, 9 May 2024 15:25:54 +0100
Subject: [PATCH 44/95] Removed more FPs from generic passwords pattern

---
 generic/patterns.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/generic/patterns.yml b/generic/patterns.yml
index b114288b..dcdaa581 100644
--- a/generic/patterns.yml
+++ b/generic/patterns.yml
@@ -14,13 +14,13 @@ patterns:
       additional_not_match:
         # something that means "password" or a placeholder name - either a variable or a placeholder
         # a literal value or config switch
-        - ^(?i)(?:[a-z0-9_.]*,\s*)?(?:str\()?[[<(]?(?:(?:(?:user|key)_?)?(?:[a-zA-Z0-9._]+[_.])?(?:the )?(?:pass?(wo?r?d|code|phrase)|pass|pwd|secret|token|tok|redacted|placeholder|dummy|pw|thephrase)|write|read|on|off|true|false|none|null( \? )?|nil|undefined|eof|ignore|eol|git|yes|no|y|n),?[\]>)]?(?:\)\s*\{)?\\?( or | \|\| )?$
+        - ^(?i)(?:[a-z0-9_.]*,\s*)?(?:str\()?[[<(]?(?:(?:(?:user|key)_?)?(?:[a-zA-Z0-9._]+[_.])?(?:the )?(?:pass?(wo?r?d|code|phrase)|pass|pwd|secret|token|tok|redacted|placeholder|dummy|pw|thephrase)|write|read|on|off|true|false|none|null( \? )?|nil|undefined|eof|ignore|eol|git|yes|no|y|n),?[\]>)]?(?:\)\s*\{)?\\?(( or | \|\| ).*)?$
         # Python type hints, Swift typing
         - ^\s*(?:(?:typing\.)?(?:(?:[Tt]uple|[Ll]ist|[Dd]ict|Callable|Iterable|Sequence|Optional|Union)\[.*|(?:int|str|float|(?:typing.)?Any|None|bytes|bool|ReadableBuffer)\s*(?:[,|].*)?|(?:Int|Swift\.Int|Int32)\.*))\s*$
         # ..., \, , \n, \0, ',' and other single chars, smilies, hex, digits, nothing at all,
         # directories, regex, format string placeholder, urllib demo passphrase, "optional" in docs, a variable substitution, or surrounded by brackets of various kinds
         # all with possible ',' and surrounding whitespace, possibly with a following comment
-        - ^\s*(?:\.\.\.|\\|\\n|\\0|\?|[,()[\]{}`.]\\?|-[)(]|\\f21b|0x[A-Fa-f0-9]+|[0-9]{1,4}|(?:~|/tmp|\.\.|\.)|\\{1,2}w\+/g,( \\?)?|%[sr]|geheim\$parole|\([Oo]ptional\).*|\$?(?:\{\{?[^}]+\}\}?|\(\(?[^)]+\)\)?|\[\[?[^\]+]\]\]?)|(before|hover)(,| \{))?,?\s*(?:\s*(?:/\*|#|//).*)?$
+        - ^\s*(?:\.\.\.|\\|\\n|\\0|\?|[,()[\]{}`.]\\?|-[)(]|\\f21b|0x[A-Fa-f0-9]+|[0-9]{1,4}|(?:~|/tmp|\.\.|\.)|\\{1,2}w\+/g,( \\?)?|%[sr]|geheim\$parole|\([Oo]ptional\).*|\$?(?:\{\{?[^}]+\}\}?|\(\(?[^)]+\)\)?|\[\[?[^\]+]\]\]?)|(before|hover|focus)(,| \{))?,?\s*(?:\s*(?:/\*|#|//).*)?$
         # function definitions, e.g. Javascript, function calls or variable declaration
         - ^(?:function\s*\([^)]*\)\s*{\s*.*|\([^)]*\)\s*=>\s*(?:{\s*|[^;)]+[;)])|(?:new |\([A-Za-z]+\)\s*)?[a-zA-Z0-9_.]+\s*\(.*|(?:public|private) [A-Za-z0-9_]+ \{|[A-Za-z0-9_.-]+\s*\) \{)$
         # reference to a member variable, index into a variable, bash variables, perl hash key index, environment vars

From e7bb31a00c58c42ef7dbea04e03449057fae7b43 Mon Sep 17 00:00:00 2001
From: aegilops <41705651+aegilops@users.noreply.github.com>
Date: Thu, 9 May 2024 15:38:06 +0100
Subject: [PATCH 45/95] Remove some FPs for YAML patterns

---
 configs/patterns.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/configs/patterns.yml b/configs/patterns.yml
index 8543216b..fd066402 100644
--- a/configs/patterns.yml
+++ b/configs/patterns.yml
@@ -100,7 +100,7 @@ patterns:
         - ^(?:keyPassphrase|password|key|[ \t]+|\$\{[^}]+}|(?:str|string|int|bool)( +#.*)?),?$
         - ^(?:.* = )?(?:None|[Tt]rue|[Ff]alse|[Nn]ull|Default(?:Type)?|Event|[A-Z]+_KEY|VERSION|NAME|update|destroy|(?:dis|en)ableEventListeners|\.\.\.),?$
         - ^(?:(?:this|self|obj)\.)(?:[A-Za-z_]+\,|[A-Za-z_].*)$
-        - ^(?:(?:[a-zA-Z_]+(?:\(\))?\.)*[a-zA-Z_]+\(\)|\|\s*)$
+        - ^(?:(?:[a-zA-Z_]+(?:\(\))?\.)*[a-zA-Z_]+\(\)|\|\s*)$|\{\{[^}]+\}\}|\$\{\{ 
         - ^\s*(?:typing\.)?(?:[Tt]uple|[Ll]ist|[Dd]ict|Callable|Iterable|Sequence|Optional|Union)\[.*$
     comments:
       - "The hardcoded password is any length"
@@ -137,7 +137,7 @@ patterns:
         - ^(?:keyPassphrase|password|key|[ \t]+|\$\{[^}]+}|(?:str|string|int|bool)( +#.*)?),?$
         - ^(?:.* = )?(?:None|[Tt]rue|[Ff]alse|[Nn]ull|Default(?:Type)?|Event|[A-Z]+_KEY|VERSION|NAME|update|destroy|(?:dis|en)ableEventListeners|\.\.\.),?$
         - '^(?:(?:(?:this|self|obj)\.)(?:[A-Za-z_]+\,|[A-Za-z_].*)|([!&*{}[\],#|>@`"''%]|[:?-] ).*)$'
-        - ^(?:[a-zA-Z_]+(?:\(\))?\.)*[a-zA-Z_]+\(\)$
+        - ^(?:[a-zA-Z_]+(?:\(\))?\.)*[a-zA-Z_]+\(\)$|\$\{\{[^}]+\}\}
         - ^\s*(?:typing\.)?(?:[Tt]uple|[Ll]ist|[Dd]ict|Callable|Iterable|Sequence|Optional|Union)\[.*$
     comments:
       - "The hardcoded password is any length"

From b129398413676d05ec373fd0c60b631d9c100035 Mon Sep 17 00:00:00 2001
From: aegilops <41705651+aegilops@users.noreply.github.com>
Date: Thu, 9 May 2024 15:48:18 +0100
Subject: [PATCH 46/95] Remove YAML external references as FP

---
 configs/patterns.yml | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/configs/patterns.yml b/configs/patterns.yml
index fd066402..36edb161 100644
--- a/configs/patterns.yml
+++ b/configs/patterns.yml
@@ -100,7 +100,7 @@ patterns:
         - ^(?:keyPassphrase|password|key|[ \t]+|\$\{[^}]+}|(?:str|string|int|bool)( +#.*)?),?$
         - ^(?:.* = )?(?:None|[Tt]rue|[Ff]alse|[Nn]ull|Default(?:Type)?|Event|[A-Z]+_KEY|VERSION|NAME|update|destroy|(?:dis|en)ableEventListeners|\.\.\.),?$
         - ^(?:(?:this|self|obj)\.)(?:[A-Za-z_]+\,|[A-Za-z_].*)$
-        - ^(?:(?:[a-zA-Z_]+(?:\(\))?\.)*[a-zA-Z_]+\(\)|\|\s*)$|\{\{[^}]+\}\}|\$\{\{ 
+        - ^(?:(?:[a-zA-Z_]+(?:\(\))?\.)*[a-zA-Z_]+\(\)|\|\s*)$|\{\{[^}]+\}\}|\$\{\{ |^!Ref 
         - ^\s*(?:typing\.)?(?:[Tt]uple|[Ll]ist|[Dd]ict|Callable|Iterable|Sequence|Optional|Union)\[.*$
     comments:
       - "The hardcoded password is any length"
@@ -163,6 +163,8 @@ patterns:
         (?:\n|\A)[ \t]*(?i)[a-z_-]*(?:secret|service_pass(wd|word|code|phrase)|pass(?:wd|word|code|phrase)?|key|token)[ \t]*:[ \t]*'
       end: |
         '([ \t]*[\r\n]|\z)
+      additional_not_match:
+        - \{\{[^{}]+\}\}
     comments:
       - "The hardcoded password is any length"
       - "Some false positives in code might appear"
@@ -187,6 +189,8 @@ patterns:
         (?:\n|\A)[ \t]*(?i)[a-z_-]*(?:secret|service_pass(wd|word|code|phrase)|pass(?:wd|word|code|phrase)?|key|token)[ \t]*:[ \t]*"
       end: |
         "([ \t]*[\r\n]|\z)
+      additional_not_match:
+        - \{\{[^{}]+\}\}
     comments:
       - "The hardcoded password is any length"
       - "Some false positives in code might appear"

From 6e2441f2e6f5e9ef0280d3f70821791c98623396 Mon Sep 17 00:00:00 2001
From: aegilops <41705651+aegilops@users.noreply.github.com>
Date: Thu, 9 May 2024 18:05:29 +0100
Subject: [PATCH 47/95] Widened to more variable names, and added more NOT
 matches

---
 generic/patterns.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/generic/patterns.yml b/generic/patterns.yml
index dcdaa581..07fbde69 100644
--- a/generic/patterns.yml
+++ b/generic/patterns.yml
@@ -8,7 +8,7 @@ patterns:
       pattern: |
         [a-zA-Z0-9!.,$%&*+?^_`{|}()[\]\\/~-][a-zA-Z0-9\t !.,$%&*+?^_`{|}()[\]\\/~-]*
       start: |
-        (?:\A|[^a-zA-Z0-9])(?i)(?:api|jwt|mysql|db)?[_.-]?(?:pass?(?:wo?r?d|code|phrase)|secret)([ \t]+As[ \t]+String)?[\t ]*(={1,3}|:)[\t ]*(?:["']|b["'])?
+        (?:\A|[^a-zA-Z0-9])(?i)(?:api|jwt|mysql|db)?[_.-]?(?:pass?(?:wo?r?d|code|phrase)|secret|key|token)([ \t]+As[ \t]+String)?[\t ]*(={1,3}|:)[\t ]*(?:["']|b["'])?
       end: |
         (\z|[\r\n'"])
       additional_not_match:
@@ -22,7 +22,7 @@ patterns:
         # all with possible ',' and surrounding whitespace, possibly with a following comment
         - ^\s*(?:\.\.\.|\\|\\n|\\0|\?|[,()[\]{}`.]\\?|-[)(]|\\f21b|0x[A-Fa-f0-9]+|[0-9]{1,4}|(?:~|/tmp|\.\.|\.)|\\{1,2}w\+/g,( \\?)?|%[sr]|geheim\$parole|\([Oo]ptional\).*|\$?(?:\{\{?[^}]+\}\}?|\(\(?[^)]+\)\)?|\[\[?[^\]+]\]\]?)|(before|hover|focus)(,| \{))?,?\s*(?:\s*(?:/\*|#|//).*)?$
         # function definitions, e.g. Javascript, function calls or variable declaration
-        - ^(?:function\s*\([^)]*\)\s*{\s*.*|\([^)]*\)\s*=>\s*(?:{\s*|[^;)]+[;)])|(?:new |\([A-Za-z]+\)\s*)?[a-zA-Z0-9_.]+\s*\(.*|(?:public|private) [A-Za-z0-9_]+ \{|[A-Za-z0-9_.-]+\s*\) \{)$
+        - ^(?:function\s*\([^)]*\)\s*{\s*.*|\([^)]*\)\s*=>\s*(?:{\s*|[^;)]+[;)])|(?:new |\([A-Za-z]+\)\s*)?[a-zA-Z0-9_.]+\s*\(.*|(?:public|private) [A-Za-z0-9_]+ \{|[A-Za-z0-9_.-]+\s*\) \{)$|\{\{[^}]+\}\}|\$\{\{ 
         # reference to a member variable, index into a variable, bash variables, perl hash key index, environment vars
         - ^\s*(?:(?:self|this)\.[a-zA-Z_][a-zA-Z0-9_.]+[,[]?|[a-zA-Z0-9_.]+\[(?:[a-zA-Z0-9_.]+)?\]?|\$(?:[1-9]|[A-Za-z0-9_]+)\{?|os\.environ\[[^\]]\]|process\.env\.[A-Z0-9_]+)\s*(?:,|\|\||&&)?\s*$
     test:

From ab428b760a888d8f19239c69e658e3b4539e1c7d Mon Sep 17 00:00:00 2001
From: aegilops <41705651+aegilops@users.noreply.github.com>
Date: Thu, 9 May 2024 18:15:29 +0100
Subject: [PATCH 48/95] Fixed test case for PagerDuty

---
 vendors/patterns.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/vendors/patterns.yml b/vendors/patterns.yml
index 8816ed04..1e0d723f 100644
--- a/vendors/patterns.yml
+++ b/vendors/patterns.yml
@@ -372,7 +372,7 @@ patterns:
       additional_not_match:
         - ^(pagerduty|pd)_(service|api)_
     test:
-      data: pd_api_key=AAAAAAAAAAAAAAAAAAAA
+      data: pd_api_key=AAAAAAAAAAAAAAAAAAAa
       start_offset: 11
     comments:
       - "Looks for surrounding context to confirm this is a PagerDuty API key, not some other 20-byte alphanumeric string"

From 145b19a11fd71f6799c2d8160c2969a34c8135e7 Mon Sep 17 00:00:00 2001
From: aegilops <41705651+aegilops@users.noreply.github.com>
Date: Thu, 9 May 2024 18:15:42 +0100
Subject: [PATCH 49/95] Fixed test cases

---
 configs/patterns.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/configs/patterns.yml b/configs/patterns.yml
index 36edb161..073b0d0d 100644
--- a/configs/patterns.yml
+++ b/configs/patterns.yml
@@ -354,7 +354,7 @@ patterns:
       - "The Base64 must contain numbers, upper case and lower case and be at least 12 characters long"
       - "Some false positives in code might appear"
     test:
-      data: "password: AAAAAAAAAAAA00=="
+      data: "password: AAAAAAAAAAAa00=="
       start_offset: 10
       end_offset: 26
 
@@ -394,7 +394,7 @@ patterns:
       pattern: |
         (([A-Za-z0-9+/]){4})+[A-Za-z0-9+/]{1,2}={0,2}
       start: |
-        [{[,][ \t]*[ \t\r\n]*"(?i)[a-z_-]*(?:secret|service_pass(wd|word|code|phrase)|pass(?:wd|word|code|phrase)?|key|token)"[ \t]*:[ \t]*"
+        [{[,][ \t]*[ \t\r\n]*"(?i)[a-z_.-]*(?:secret|service_pass(wd|word|code|phrase)|pass(?:wd|word|code|phrase)?|key|token)"[ \t]*:[ \t]*"
       end: |
         "[ \t\r\n]*[,}\]]
       additional_match:
@@ -408,7 +408,7 @@ patterns:
       - "This will not match some isolated fragments of JSON, so be aware of that when testing it"
     test:
       data: |
-        {"password": "AAAAAAAAAAAA00==", "user": "admin"}
+        {"password": "AAAAAAAAAAAa00==", "user": "admin"}
       start_offset: 14
       end_offset: 30
 

From fec8417b3c4e04f16645ba5bb585732a44a64714 Mon Sep 17 00:00:00 2001
From: aegilops <41705651+aegilops@users.noreply.github.com>
Date: Thu, 9 May 2024 18:16:11 +0100
Subject: [PATCH 50/95] More variables for generic, plus new token variants for
 generic pw

---
 generic/patterns.yml | 46 +++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 43 insertions(+), 3 deletions(-)

diff --git a/generic/patterns.yml b/generic/patterns.yml
index 07fbde69..cdba1692 100644
--- a/generic/patterns.yml
+++ b/generic/patterns.yml
@@ -8,7 +8,7 @@ patterns:
       pattern: |
         [a-zA-Z0-9!.,$%&*+?^_`{|}()[\]\\/~-][a-zA-Z0-9\t !.,$%&*+?^_`{|}()[\]\\/~-]*
       start: |
-        (?:\A|[^a-zA-Z0-9])(?i)(?:api|jwt|mysql|db)?[_.-]?(?:pass?(?:wo?r?d|code|phrase)|secret|key|token)([ \t]+As[ \t]+String)?[\t ]*(={1,3}|:)[\t ]*(?:["']|b["'])?
+        (?:\A|[^a-zA-Z0-9])(?i)(?:api|auth[a-z]+|jwt|mysql|db)?[_.-]?(?:pass?(?:wo?r?d|code|phrase)|secret|key|token)([_-][a-z+])?([ \t]+As[ \t]+String)?[\t ]*(={1,3}|:)[\t ]*(?:["']|b["'])?
       end: |
         (\z|[\r\n'"])
       additional_not_match:
@@ -45,6 +45,48 @@ patterns:
       - "String with a number of chars until a breaking char"
       - "Not matching variables, placeholders or common configuration constants such as 'read' and 'write'"
 
+
+  - name: Generic Password with hex encoded secrets
+    type: generic_passwords_hex
+    regex:
+      version: 0.1
+      pattern: |
+        [0-9a-f]{32}|[0-9a-f]{40}|[0-9a-f]{64}
+      start: |
+        (?:\A|[^a-zA-Z0-9])(?i)(?:api|auth[a-z]+|jwt|mysql|db)?[_.-]?(?:pass?(?:wo?r?d|code|phrase)|secret|key|token)([_-][a-z+])?([ \t]+As[ \t]+String)?[\t ]*(={1,3}|:)[\t ]*(?:["']|b["'])?
+      end: |
+        (\z|[\r\n'"])
+    test:
+      data: password=0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
+      start_offset: 9
+      end_offset: -1
+    comments:
+      - "`password`, `secret`, `key`, or password like prefix (fuzzy)"
+      - "Delimiters like `=` or `:` (with padding)"
+      - "Has to be a token-like value"
+
+
+  - name: Generic Password with Base64 encoded secrets
+    type: generic_passwords_base64
+    regex:
+      version: 0.1
+      pattern: |
+        (([A-Za-z0-9+/]){4})+[A-Za-z0-9+/]{1,2}={0,2}
+      start: |
+        (?:\A|[^a-zA-Z0-9])(?i)(?:api|auth[a-z]+|jwt|mysql|db)?[_.-]?(?:pass?(?:wo?r?d|code|phrase)|secret|key|token)([_-][a-z+])?([ \t]+As[ \t]+String)?[\t ]*(={1,3}|:)[\t ]*(?:["']|b["'])?
+      end: |
+        (\z|[\r\n'"])
+      additional_match:
+        - '[0-9]'
+        - '[A-Z]'
+        - '[a-z]'
+        - '^.{12,}$'
+    comments:
+      - "The Base64 must contain numbers, upper case and lower case and be at least 12 characters long"
+      - "`password`, `secret`, `key`, or password like prefix (fuzzy)"
+      - "Delimiters like `=` or `:` (with padding)"
+
+
   - name: UUIDs
     type: uuids
     regex:
@@ -60,12 +102,10 @@ patterns:
         - ^00000000-0000-0000-0000-000000000000$
         - ^(?i)00010203-0405-0607-0809-0a0b0c0d0e0f$
         - ^(?i)12345678-1234-1234-1234-123456789abc$
-
     test:
       - data: 10203040-5060-7080-90a0-b0c0d0e0f000
         start_offset: 0
         end_offset: -1
-
     expected:
       - name: uuids.txt
         start_offset: 12

From c33c470f532657b437c4d6e33755a71664252717 Mon Sep 17 00:00:00 2001
From: aegilops <41705651+aegilops@users.noreply.github.com>
Date: Fri, 10 May 2024 12:29:55 +0100
Subject: [PATCH 51/95] Updated README.md

---
 README.md         |   4 ++
 configs/README.md |  34 +++++++++++--
 generic/README.md | 125 ++++++++++++++++++++++++++++++++++++++++++++--
 3 files changed, 155 insertions(+), 8 deletions(-)

diff --git a/README.md b/README.md
index 458bdb75..aa914f03 100644
--- a/README.md
+++ b/README.md
@@ -57,6 +57,10 @@ Custom Secret Scanning Patterns repository.
 
 - Generic Passwords
 
+- Generic Password with hex encoded secrets
+
+- Generic Password with Base64 encoded secrets
+
 - UUIDs
 
 - Bearer Tokens
diff --git a/configs/README.md b/configs/README.md
index 26744eb7..4c41bb7a 100644
--- a/configs/README.md
+++ b/configs/README.md
@@ -183,7 +183,7 @@ Add these additional matches to the [Secret Scanning Custom Pattern](https://doc
 - Not Match:
 
   ```regex
-  ^(?:(?:[a-zA-Z_]+(?:\(\))?\.)*[a-zA-Z_]+\(\)|\|\s*)$
+  ^(?:(?:[a-zA-Z_]+(?:\(\))?\.)*[a-zA-Z_]+\(\)|\|\s*)$|\{\{[^}]+\}\}|\$\{\{ |^!Ref
   ```
 - Not Match:
 
@@ -261,7 +261,7 @@ Add these additional matches to the [Secret Scanning Custom Pattern](https://doc
 - Not Match:
 
   ```regex
-  ^(?:[a-zA-Z_]+(?:\(\))?\.)*[a-zA-Z_]+\(\)$
+  ^(?:[a-zA-Z_]+(?:\(\))?\.)*[a-zA-Z_]+\(\)$|\$\{\{[^}]+\}\}
   ```
 - Not Match:
 
@@ -315,6 +315,20 @@ _version: v0.1_
 
 </details>
 
+<details>
+<summary>Additional Matches</summary>
+
+Add these additional matches to the [Secret Scanning Custom Pattern](https://docs.github.com/en/enterprise-cloud@latest/code-security/secret-scanning/defining-custom-patterns-for-secret-scanning#example-of-a-custom-pattern-specified-using-additional-requirements).
+
+
+- Not Match:
+
+  ```regex
+  \{\{[^{}]+\}\}
+  ```
+
+</details>
+
 ## YAML hardcoded passwords (double quoted strings)
 
 **⚠️ WARNING: THIS RULE IS EXPERIMENTAL AND MIGHT CAUSE A HIGH FALSE POSITIVE RATE (test before commiting to org level) ⚠️**
@@ -359,6 +373,20 @@ _version: v0.1_
 
 </details>
 
+<details>
+<summary>Additional Matches</summary>
+
+Add these additional matches to the [Secret Scanning Custom Pattern](https://docs.github.com/en/enterprise-cloud@latest/code-security/secret-scanning/defining-custom-patterns-for-secret-scanning#example-of-a-custom-pattern-specified-using-additional-requirements).
+
+
+- Not Match:
+
+  ```regex
+  \{\{[^{}]+\}\}
+  ```
+
+</details>
+
 ## YAML hardcoded passwords (multiline strings)
 
 **⚠️ WARNING: THIS RULE IS EXPERIMENTAL AND MIGHT CAUSE A HIGH FALSE POSITIVE RATE (test before commiting to org level) ⚠️**
@@ -761,7 +789,7 @@ _version: v0.1_
 <summary>Start Pattern</summary>
 
 ```regex
-[{[,][ \t]*[ \t\r\n]*"(?i)[a-z_-]*(?:secret|service_pass(wd|word|code|phrase)|pass(?:wd|word|code|phrase)?|key|token)"[ \t]*:[ \t]*"
+[{[,][ \t]*[ \t\r\n]*"(?i)[a-z_.-]*(?:secret|service_pass(wd|word|code|phrase)|pass(?:wd|word|code|phrase)?|key|token)"[ \t]*:[ \t]*"
 ```
 
 </details><details>
diff --git a/generic/README.md b/generic/README.md
index dd195f4d..1875d1b3 100644
--- a/generic/README.md
+++ b/generic/README.md
@@ -9,7 +9,7 @@
 
 
 
-_version: v0.4_
+_version: v0.5_
 
 **Comments / Notes:**
 
@@ -36,7 +36,7 @@ _version: v0.4_
 <summary>Start Pattern</summary>
 
 ```regex
-(?:\A|[^a-zA-Z0-9])(?i)(?:api|jwt|mysql|db)?[_.-]?(?:pass?(?:wo?r?d|code|phrase)|secret)([ \t]+As[ \t]+String)?[\t ]*(={1,3}|:)[\t ]*(?:["']|b["'])?
+(?:\A|[^a-zA-Z0-9])(?i)(?:api|auth[a-z]+|jwt|mysql|db)?[_.-]?(?:pass?(?:wo?r?d|code|phrase)|secret|key|token)([_-][a-z0-9]+){0,3}([ \t]+As[ \t]+String)?[\t ]*(={1,3}|:)[\t ]*(?:["']|b["'])?
 ```
 
 </details><details>
@@ -57,7 +57,7 @@ Add these additional matches to the [Secret Scanning Custom Pattern](https://doc
 - Not Match:
 
   ```regex
-  ^(?i)(?:[a-z0-9_.]*,\s*)?(?:str\()?[[<(]?(?:(?:(?:user|key)_?)?(?:[a-zA-Z0-9._]+[_.])?(?:the )?(?:pass?(wo?r?d|code|phrase)|pass|pwd|secret|token|tok|redacted|placeholder|dummy|pw|thephrase)|write|read|on|off|true|false|none|null|nil|undefined|eof|ignore|eol|git|yes|no|y|n),?[\]>)]?(?:\)\s*\{)?\\?( or )?$
+  _?)?(?:[a-zA-Z0-9._]+[_.])?(?:the )?(?:pass?(wo?r?d|code|phrase)|pass|pwd|secret|token|tok|redacted|placeholder|dummy|pw|thephrase)|write|read|on|off|true|false|none|null( \? )?|nil|undefined|eof|ignore|eol|git|yes|no|y|n),?\s*\){0,2}[\]>)]?(?:\)\s*\{)?\\?(( or | \|\| ).*)?$
   ```
 - Not Match:
 
@@ -67,12 +67,12 @@ Add these additional matches to the [Secret Scanning Custom Pattern](https://doc
 - Not Match:
 
   ```regex
-  ^\s*(?:\.\.\.|\\|\\n|\\0|[,()[\]{}`.]\\?|-[)(]|0x[A-Fa-f0-9]+|[0-9]{1,4}|(?:~|/tmp|\.\.|\.)|\\{1,2}w\+/g,( \\?)?|%[sr]|geheim\$parole|\([Oo]ptional\).*|\$?(?:\{\{?[^}]+\}\}?|\(\(?[^)]+\)\)?|\[\[?[^\]+]\]\]?))?,?\s*(?:\s*(?:/\*|#|//).*)?$
+  ^\s*(?:\.\.\.|\\|\\n|\\0|\?|\$\(|[,()[\]{}`.]\\?|-[)(]|\\f21b|0x[A-Fa-f0-9]+|[0-9]{1,4}|(?:~|/tmp|\.\.|\.)|\\{1,2}w\+/g,( \\?)?|%[sr]|geheim\$parole|\([Oo]ptional\).*|\$?(?:\{\{?[^}]+\}\}?|\(\(?[^)]+\)\)?|\[\[?[^\]+]\]\]?)|(before|hover|focus)(,| \{))?,?\s*(?:\s*(?:/\*|#|//).*)?$
   ```
 - Not Match:
 
   ```regex
-  ^(?:function\s*\([^)]*\)\s*{\s*.*|\([^)]*\)\s*=>\s*(?:{\s*|[^;)]+[;)])|(?:new )?[a-zA-Z0-9_.]+\(.*|(?:public|private) [A-Za-z0-9_]+ \{)$
+  ^(?:function\s*\([^)]*\)\s*{\s*.*|\([^)]*\)\s*=>\s*(?:{\s*|[^;)]+[;)])|(?:new |\([A-Za-z]+\)\s*)?[a-zA-Z0-9_.]+\s*\(.*|(?:public|private) [A-Za-z0-9_]+ \{|[A-Za-z0-9_.-]+\s*\) \{)$|\{\{[^}]+\}\}|\$\{\{
   ```
 - Not Match:
 
@@ -82,6 +82,121 @@ Add these additional matches to the [Secret Scanning Custom Pattern](https://doc
 
 </details>
 
+## Generic Password with hex encoded secrets
+
+
+
+_version: v0.1_
+
+**Comments / Notes:**
+
+
+- `password`, `secret`, `key`, or password like prefix (fuzzy)
+
+- Delimiters like `=` or `:` (with padding)
+
+- Has to be a token-like value
+  
+
+<details>
+<summary>Pattern Format</summary>
+
+```regex
+[0-9a-f]{32}|[0-9a-f]{40}|[0-9a-f]{64}
+```
+
+</details>
+
+<details>
+<summary>Start Pattern</summary>
+
+```regex
+(?:\A|[^a-zA-Z0-9])(?i)(?:api|auth[a-z]+|jwt|mysql|db)?[_.-]?(?:pass?(?:wo?r?d|code|phrase)|secret|key|token)([_-][a-z0-9]+){0,3}([ \t]+As[ \t]+String)?[\t ]*(={1,3}|:)[\t ]*(?:["']|b["'])?
+```
+
+</details><details>
+<summary>End Pattern</summary>
+
+```regex
+(\z|[\r\n'"])
+```
+
+</details>
+
+## Generic Password with Base64 encoded secrets
+
+
+
+_version: v0.1_
+
+**Comments / Notes:**
+
+
+- The Base64 must contain numbers, upper case and lower case and be at least 12 characters long
+
+- `password`, `secret`, `key`, or password like prefix (fuzzy)
+
+- Delimiters like `=` or `:` (with padding)
+  
+
+<details>
+<summary>Pattern Format</summary>
+
+```regex
+(([A-Za-z0-9+/]){4})+[A-Za-z0-9+/]{1,2}={0,2}
+```
+
+</details>
+
+<details>
+<summary>Start Pattern</summary>
+
+```regex
+(?:\A|[^a-zA-Z0-9])(?i)(?:api|auth[a-z]+|jwt|mysql|db)?[_.-]?(?:pass?(?:wo?r?d|code|phrase)|secret|key|token)([_-][a-z0-9]+){0,3}([ \t]+As[ \t]+String)?[\t ]*(={1,3}|:)[\t ]*(?:["']|b["'])?
+```
+
+</details><details>
+<summary>End Pattern</summary>
+
+```regex
+(\z|[\r\n'"])
+```
+
+</details>
+
+<details>
+<summary>Additional Matches</summary>
+
+Add these additional matches to the [Secret Scanning Custom Pattern](https://docs.github.com/en/enterprise-cloud@latest/code-security/secret-scanning/defining-custom-patterns-for-secret-scanning#example-of-a-custom-pattern-specified-using-additional-requirements).
+
+
+
+- Match:
+
+  ```regex
+  [0-9]
+  ```
+
+- Match:
+
+  ```regex
+  [A-Z]
+  ```
+
+- Match:
+
+  ```regex
+  [a-z]
+  ```
+
+- Match:
+
+  ```regex
+  ^.{12,}$
+  ```
+
+</details>
+
 ## UUIDs
 
 

From 2be7749a1356b810401590fd21e8ffd6ac71bd52 Mon Sep 17 00:00:00 2001
From: aegilops <41705651+aegilops@users.noreply.github.com>
Date: Fri, 10 May 2024 17:53:19 +0100
Subject: [PATCH 52/95] Added test data for full db c/s

---
 database/patterns.yml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/database/patterns.yml b/database/patterns.yml
index b90bce3e..c8185f68 100644
--- a/database/patterns.yml
+++ b/database/patterns.yml
@@ -23,6 +23,11 @@ patterns:
       additional_not_match:
         # placeholders, variable substitutions, etc.
         - (^|;)([Pp]assword|[Pp]wd|[Ss]hared[Ss]ecret[Vv]alue|[Aa]ccount[Kk]ey|PW|pw|[Cc]ipher [Kk]ey|OAuth Access Token Secret)=(%s\b|\{\{[^}]+\}\}|\{[0-9]+\}|\$?\{[^}]+\}|\[[A-Z_]+\]|['"`]|$)
+    test:
+      data: connection_string = "Data Source=eee,1433;Initial Catalog=bbb;Integrated Security=False;User ID=ccc@aaa;Password=ddd;"
+      start_offset: 21
+      end_offset: 117
+
 
   - name: Database Connection String (1)
     type: database_connection_string_1

From fbeeb4cf71b35afded528c82e9c9bce237a275dc Mon Sep 17 00:00:00 2001
From: aegilops <41705651+aegilops@users.noreply.github.com>
Date: Fri, 10 May 2024 17:53:55 +0100
Subject: [PATCH 53/95] Added test data for generic base64 pw

---
 generic/patterns.yml | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/generic/patterns.yml b/generic/patterns.yml
index cdba1692..d707464d 100644
--- a/generic/patterns.yml
+++ b/generic/patterns.yml
@@ -8,19 +8,19 @@ patterns:
       pattern: |
         [a-zA-Z0-9!.,$%&*+?^_`{|}()[\]\\/~-][a-zA-Z0-9\t !.,$%&*+?^_`{|}()[\]\\/~-]*
       start: |
-        (?:\A|[^a-zA-Z0-9])(?i)(?:api|auth[a-z]+|jwt|mysql|db)?[_.-]?(?:pass?(?:wo?r?d|code|phrase)|secret|key|token)([_-][a-z+])?([ \t]+As[ \t]+String)?[\t ]*(={1,3}|:)[\t ]*(?:["']|b["'])?
+        (?:\A|[^a-zA-Z0-9])(?i)(?:api|auth[a-z]+|jwt|mysql|db)?[_.-]?(?:pass?(?:wo?r?d|code|phrase)|secret|key|token)([_-][a-z0-9]+){0,3}([ \t]+As[ \t]+String)?[\t ]*(={1,3}|:)[\t ]*(?:["']|b["'])?
       end: |
         (\z|[\r\n'"])
       additional_not_match:
         # something that means "password" or a placeholder name - either a variable or a placeholder
         # a literal value or config switch
-        - ^(?i)(?:[a-z0-9_.]*,\s*)?(?:str\()?[[<(]?(?:(?:(?:user|key)_?)?(?:[a-zA-Z0-9._]+[_.])?(?:the )?(?:pass?(wo?r?d|code|phrase)|pass|pwd|secret|token|tok|redacted|placeholder|dummy|pw|thephrase)|write|read|on|off|true|false|none|null( \? )?|nil|undefined|eof|ignore|eol|git|yes|no|y|n),?[\]>)]?(?:\)\s*\{)?\\?(( or | \|\| ).*)?$
+        - _?)?(?:[a-zA-Z0-9._]+[_.])?(?:the )?(?:pass?(wo?r?d|code|phrase)|pass|pwd|secret|token|tok|redacted|placeholder|dummy|pw|thephrase)|write|read|on|off|true|false|none|null( \? )?|nil|undefined|eof|ignore|eol|git|yes|no|y|n),?\s*\){0,2}[\]>)]?(?:\)\s*\{)?\\?(( or | \|\| ).*)?$
         # Python type hints, Swift typing
         - ^\s*(?:(?:typing\.)?(?:(?:[Tt]uple|[Ll]ist|[Dd]ict|Callable|Iterable|Sequence|Optional|Union)\[.*|(?:int|str|float|(?:typing.)?Any|None|bytes|bool|ReadableBuffer)\s*(?:[,|].*)?|(?:Int|Swift\.Int|Int32)\.*))\s*$
         # ..., \, , \n, \0, ',' and other single chars, smilies, hex, digits, nothing at all,
         # directories, regex, format string placeholder, urllib demo passphrase, "optional" in docs, a variable substitution, or surrounded by brackets of various kinds
         # all with possible ',' and surrounding whitespace, possibly with a following comment
-        - ^\s*(?:\.\.\.|\\|\\n|\\0|\?|[,()[\]{}`.]\\?|-[)(]|\\f21b|0x[A-Fa-f0-9]+|[0-9]{1,4}|(?:~|/tmp|\.\.|\.)|\\{1,2}w\+/g,( \\?)?|%[sr]|geheim\$parole|\([Oo]ptional\).*|\$?(?:\{\{?[^}]+\}\}?|\(\(?[^)]+\)\)?|\[\[?[^\]+]\]\]?)|(before|hover|focus)(,| \{))?,?\s*(?:\s*(?:/\*|#|//).*)?$
+        - ^\s*(?:\.\.\.|\\|\\n|\\0|\?|\$\(|[,()[\]{}`.]\\?|-[)(]|\\f21b|0x[A-Fa-f0-9]+|[0-9]{1,4}|(?:~|/tmp|\.\.|\.)|\\{1,2}w\+/g,( \\?)?|%[sr]|geheim\$parole|\([Oo]ptional\).*|\$?(?:\{\{?[^}]+\}\}?|\(\(?[^)]+\)\)?|\[\[?[^\]+]\]\]?)|(before|hover|focus)(,| \{))?,?\s*(?:\s*(?:/\*|#|//).*)?$
         # function definitions, e.g. Javascript, function calls or variable declaration
         - ^(?:function\s*\([^)]*\)\s*{\s*.*|\([^)]*\)\s*=>\s*(?:{\s*|[^;)]+[;)])|(?:new |\([A-Za-z]+\)\s*)?[a-zA-Z0-9_.]+\s*\(.*|(?:public|private) [A-Za-z0-9_]+ \{|[A-Za-z0-9_.-]+\s*\) \{)$|\{\{[^}]+\}\}|\$\{\{ 
         # reference to a member variable, index into a variable, bash variables, perl hash key index, environment vars
@@ -53,7 +53,7 @@ patterns:
       pattern: |
         [0-9a-f]{32}|[0-9a-f]{40}|[0-9a-f]{64}
       start: |
-        (?:\A|[^a-zA-Z0-9])(?i)(?:api|auth[a-z]+|jwt|mysql|db)?[_.-]?(?:pass?(?:wo?r?d|code|phrase)|secret|key|token)([_-][a-z+])?([ \t]+As[ \t]+String)?[\t ]*(={1,3}|:)[\t ]*(?:["']|b["'])?
+        (?:\A|[^a-zA-Z0-9])(?i)(?:api|auth[a-z]+|jwt|mysql|db)?[_.-]?(?:pass?(?:wo?r?d|code|phrase)|secret|key|token)([_-][a-z0-9]+){0,3}([ \t]+As[ \t]+String)?[\t ]*(={1,3}|:)[\t ]*(?:["']|b["'])?
       end: |
         (\z|[\r\n'"])
     test:
@@ -73,7 +73,7 @@ patterns:
       pattern: |
         (([A-Za-z0-9+/]){4})+[A-Za-z0-9+/]{1,2}={0,2}
       start: |
-        (?:\A|[^a-zA-Z0-9])(?i)(?:api|auth[a-z]+|jwt|mysql|db)?[_.-]?(?:pass?(?:wo?r?d|code|phrase)|secret|key|token)([_-][a-z+])?([ \t]+As[ \t]+String)?[\t ]*(={1,3}|:)[\t ]*(?:["']|b["'])?
+        (?:\A|[^a-zA-Z0-9])(?i)(?:api|auth[a-z]+|jwt|mysql|db)?[_.-]?(?:pass?(?:wo?r?d|code|phrase)|secret|key|token)([_-][a-z0-9]+){0,3}([ \t]+As[ \t]+String)?[\t ]*(={1,3}|:)[\t ]*(?:["']|b["'])?
       end: |
         (\z|[\r\n'"])
       additional_match:
@@ -81,6 +81,10 @@ patterns:
         - '[A-Z]'
         - '[a-z]'
         - '^.{12,}$'
+    test:
+      data: password="AAAAAAAAAAAa00=="
+      start_offset: 10
+      end_offset: 26
     comments:
       - "The Base64 must contain numbers, upper case and lower case and be at least 12 characters long"
       - "`password`, `secret`, `key`, or password like prefix (fuzzy)"
@@ -124,7 +128,7 @@ patterns:
       pattern: |
         [a-zA-Z0-9_.=/+:-]+
       start: |
-        \b([Bb]earer|[Tt]oken)[ ]+
+        \b([Bb]earer |[Tt]oken (token=)?)
       end: |
         \z|[\s'"]
       additional_not_match:
@@ -148,6 +152,7 @@ patterns:
       - "As used in an Authorization header"
       - "We try to remove common placeholders"
 
+
   - name: OAuth client secret and ID pair
     type: oauth_client_secret
     regex:

From c010a44bc5af3e0fda420f50a2da62b687fbefc3 Mon Sep 17 00:00:00 2001
From: aegilops <41705651+aegilops@users.noreply.github.com>
Date: Fri, 10 May 2024 17:54:06 +0100
Subject: [PATCH 54/95] Updated README.md

---
 generic/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/generic/README.md b/generic/README.md
index 1875d1b3..6633c3a6 100644
--- a/generic/README.md
+++ b/generic/README.md
@@ -286,7 +286,7 @@ _version: v0.2_
 <summary>Start Pattern</summary>
 
 ```regex
-\b([Bb]earer|[Tt]oken)[ ]+
+\b([Bb]earer |[Tt]oken (token=)?)
 ```
 
 </details><details>

From b6261d243912aa370b72018a3029e661760eb8d2 Mon Sep 17 00:00:00 2001
From: aegilops <41705651+aegilops@users.noreply.github.com>
Date: Wed, 15 May 2024 15:35:07 +0100
Subject: [PATCH 55/95] Fewer FPs generic pw, plus misc updates

---
 configs/patterns.yml |  6 ++++--
 generic/patterns.yml | 43 ++++++++++++++++++++++++++++++++++++++-----
 2 files changed, 42 insertions(+), 7 deletions(-)

diff --git a/configs/patterns.yml b/configs/patterns.yml
index 073b0d0d..bbf71dd2 100644
--- a/configs/patterns.yml
+++ b/configs/patterns.yml
@@ -89,7 +89,7 @@ patterns:
       Pattern to find hardcoded passwords in YAML configuration files
     experimental: true
     regex:
-      version: 0.1
+      version: 0.2
       pattern: |
         [^\r\n`'"\x00-\x08]+
       start: |
@@ -103,6 +103,7 @@ patterns:
         - ^(?:(?:[a-zA-Z_]+(?:\(\))?\.)*[a-zA-Z_]+\(\)|\|\s*)$|\{\{[^}]+\}\}|\$\{\{ |^!Ref 
         - ^\s*(?:typing\.)?(?:[Tt]uple|[Ll]ist|[Dd]ict|Callable|Iterable|Sequence|Optional|Union)\[.*$
     comments:
+      - "Expect large numbers of false positives on variables containing 'key' or 'token'"
       - "The hardcoded password is any length"
       - "Some false positives in code might appear"
       - "The pattern only checks for certain key words to end the variable name (`secret`, `password`, etc.)"
@@ -431,7 +432,8 @@ patterns:
         - '[a-f]'
     comments:
       - "The hex token must be 32, 40 or 64 characters long, and contain numbers and letters"
-      - "Some false positives in code might appear"
+      - "This may match in code, such as Python, that resembles JSON"
+      - "This will not match some isolated fragments of JSON, so be aware of that when testing it"
     test:
       data: |
         {"password": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", "user": "admin"}
diff --git a/generic/patterns.yml b/generic/patterns.yml
index d707464d..82b394f3 100644
--- a/generic/patterns.yml
+++ b/generic/patterns.yml
@@ -3,12 +3,13 @@ name: Generic Secrets / Passwords
 patterns:
   - name: Generic Passwords
     type: generic_passwords
+    experimental: true
     regex:
       version: 0.5
       pattern: |
         [a-zA-Z0-9!.,$%&*+?^_`{|}()[\]\\/~-][a-zA-Z0-9\t !.,$%&*+?^_`{|}()[\]\\/~-]*
       start: |
-        (?:\A|[^a-zA-Z0-9])(?i)(?:api|auth[a-z]+|jwt|mysql|db)?[_.-]?(?:pass?(?:wo?r?d|code|phrase)|secret|key|token)([_-][a-z0-9]+){0,3}([ \t]+As[ \t]+String)?[\t ]*(={1,3}|:)[\t ]*(?:["']|b["'])?
+        (?:\A|[^a-zA-Z0-9])(?i)[a-z0-9_.-]*(?:api|auth[a-z]+|jwt|mysql|db)?[_.-]?(?:pass?(?:wo?r?d|code|phrase)|secret|key|token)([_-][a-z0-9]+){0,3}([ \t]+As[ \t]+String)?[\t ]*(={1,3}|:)[\t ]*(?:["']|b["'])?
       end: |
         (\z|[\r\n'"])
       additional_not_match:
@@ -40,12 +41,43 @@ patterns:
         start_offset: 97
         end_offset: 112
     comments:
+      - "Likely to cause large numbers of false positives - use with caution"
       - "`password`, `secret`, `key`, or password like prefix (fuzzy)"
       - "Delimiters like `=` or `:` (with padding)"
       - "String with a number of chars until a breaking char"
       - "Not matching variables, placeholders or common configuration constants such as 'read' and 'write'"
 
 
+  - name: Generic Passwords (fewer FPs)
+    type: generic_passwords_fewer_fps
+    regex:
+      version: 0.1
+      pattern: |
+        ((?i)[a-z0-9_.-]*(api|auth[a-z]*|jwt|mysql|db)[_.-]?)?((?i)pass?(wo?r?d|code|phrase)|secret|token|key)([_-][A-Za-z0-9]+){0,4}_{0,2}(["'`]|[ \t]+As[ \t]+String)?[\t ]*(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)[\t ]*([br]?"[A-Za-z0-9_/+.!-]+"|[br]?'[A-Za-z0-9_/+.!-]+')
+      start: |
+        \A|[^0-9A-Za-z]
+      end: |
+        \z|[^A-Za-z0-9]
+      additional_not_match:
+        - |-
+          ^[A-Za-z0-9_.-]*(key|KEY|[Tt]oken|TOKEN)(_[a-zA-Z]+)?['"]?\s*([:=]|=>)\s*["']([Ee]mploye[er]|[Ss]taff|([Ss]earch)?[Rr]esult|[a-z][a-zA-Z]+CSX[A-Z][A-Za-z]+|[A-Za-z]*[Bb]ase(64|32|58)|object|claret|assigns?|clean|contains|error|expand|generate|hoist|indent(ation)?|invert|jumps?|pairs?|param(eter)?s?|pop|rewrite|temp(orary)?|token(s|i[sz]e)?|type|((un)?(quote|shift|wrap|finished))|[a-z]{2,10}([A-Z][a-z]{1,15}){1,6}|(compile|is|has|make|add|each|check|close|cache|format|tag|get|set)([A-Z][A-Za-z]+)?|gadget|classic|(try_)?(base|mode|grade|model)|words|identifier|[a-z.-]+\.(jpe?g|(x|ht)ml|txt|docx?|xlsx?|pdf|png)|enabled|name\.invalidPattern|\.|\.data-api|expect|file|config|ansi|Default(Type)?|Cache-Control|((notD|d)eepE|e)qual|name|NAME|package|version|VERSION|start|end|step|async|Event|throws|ok|notOK|verbose|push(Result)?|slimAssertions|(p|notP)ropEqual|((notS|s)trict|not)Equal|value|prev|next|year|key[0-9]?|destroy|[a-z]+EventListeners|timeout|str(ing)?|hmac|uuid|update|find|true|false|[Nn][ui]ll|[Nn]one|[a-z_]+\.((tf)?state|id|key)|(hibernate|ws|err|i18n|employee|bs|org|com|sun|java)(\.[a-zA-Z0-9_]+){1,4}|[A-Z_]+_KEY)["']$
+        - |-
+          (?i)(token|key)[_-](name|format|type|enabled|success|type|method)\b
+        - |-
+          ^token(_[A-Z]+)?['"]?\s*[:=]\s*['"](barline|parenthesis|qualified|suport|symbol|statementEnd|singleLineTitle|character|pageBreak|operator|optionalTitle|option|zupfnoter|chordname|macro|error|escape|indent|term|titleUnderline|tag|link|literal|(other|table)Block|list|value|control|set|support|injections|array|doc|source|heading|tokens|storage|empty|newline|empty_line|keyword|comment|meta|[lr]?paren|class|punctuation|regexp?|constant|string|entity|invalid|support|variable|multiline|language|paren|markup|singleline|nospell|text|array|doc|source|heading|tokens)(\.{1,2}[A-Za-z0-9_-]+){0,6}[!.]?["']$
+        - "^KEY_[A-Z]+[0-9]{0,3}: 'k[a-zA-Z0-9]{1,6}'$"
+    test:
+      data: password="Password123"
+      start_offset: 0
+      end_offset: 22
+    comments:
+      - "Expect many false positives if run against vendored-in code, such as JS libraries - use with caution"
+      - "`password`, `secret`, `key`, `token` etc. password-like prefix (fuzzy)"
+      - "Delimiters like `=` or `:` (with padding)"
+      - "Matches fewer characters (A-Za-z0-9_/+.!-), and requires matching quotes around the string"
+      - "Attempts to remove variables, placeholders, and common configuration constants such as 'read' and 'write'"
+
+
   - name: Generic Password with hex encoded secrets
     type: generic_passwords_hex
     regex:
@@ -53,7 +85,7 @@ patterns:
       pattern: |
         [0-9a-f]{32}|[0-9a-f]{40}|[0-9a-f]{64}
       start: |
-        (?:\A|[^a-zA-Z0-9])(?i)(?:api|auth[a-z]+|jwt|mysql|db)?[_.-]?(?:pass?(?:wo?r?d|code|phrase)|secret|key|token)([_-][a-z0-9]+){0,3}([ \t]+As[ \t]+String)?[\t ]*(={1,3}|:)[\t ]*(?:["']|b["'])?
+        (?:\A|[^a-zA-Z0-9])(?i)[a-z0-9._-]*(?:api|auth[a-z]+|jwt|mysql|db)?[_.-]?(?:pass?(?:wo?r?d|code|phrase)|secret|key|token)([_-][a-z0-9]+){0,3}([ \t]+As[ \t]+String)?[\t ]*(={1,3}|:)[\t ]*(?:["']|b["'])?
       end: |
         (\z|[\r\n'"])
     test:
@@ -63,7 +95,7 @@ patterns:
     comments:
       - "`password`, `secret`, `key`, or password like prefix (fuzzy)"
       - "Delimiters like `=` or `:` (with padding)"
-      - "Has to be a token-like value"
+      - "Has to be a token-like value - a 32, 40 or 64 character hex string"
 
 
   - name: Generic Password with Base64 encoded secrets
@@ -73,7 +105,7 @@ patterns:
       pattern: |
         (([A-Za-z0-9+/]){4})+[A-Za-z0-9+/]{1,2}={0,2}
       start: |
-        (?:\A|[^a-zA-Z0-9])(?i)(?:api|auth[a-z]+|jwt|mysql|db)?[_.-]?(?:pass?(?:wo?r?d|code|phrase)|secret|key|token)([_-][a-z0-9]+){0,3}([ \t]+As[ \t]+String)?[\t ]*(={1,3}|:)[\t ]*(?:["']|b["'])?
+        (?:\A|[^a-zA-Z0-9])(?i)[a-z0-9._-]*(?:api|auth[a-z]+|jwt|mysql|db)?[_.-]?(?:pass?(?:wo?r?d|code|phrase)|secret|key|token)([_-][a-z0-9]+){0,3}([ \t]+As[ \t]+String)?[\t ]*(={1,3}|:)[\t ]*(?:["']|b["'])?
       end: |
         (\z|[\r\n'"])
       additional_match:
@@ -121,6 +153,7 @@ patterns:
         start_offset: 131
         end_offset: 167
 
+
   - name: Bearer Tokens
     type: bearer_tokens
     regex:
@@ -128,7 +161,7 @@ patterns:
       pattern: |
         [a-zA-Z0-9_.=/+:-]+
       start: |
-        \b([Bb]earer |[Tt]oken (token=)?)
+        (Authorization: |['"])([Bb]earer |[Tt]oken (token=)?)
       end: |
         \z|[\s'"]
       additional_not_match:

From 48f353d451ed0719291560bb6fe797f7d9f4ecab Mon Sep 17 00:00:00 2001
From: aegilops <41705651+aegilops@users.noreply.github.com>
Date: Wed, 15 May 2024 15:36:39 +0100
Subject: [PATCH 56/95] Updated README.md

---
 README.md         |  2 +-
 configs/README.md |  8 +++--
 generic/README.md | 88 +++++++++++++++++++++++++++++++++++++++++++----
 3 files changed, 89 insertions(+), 9 deletions(-)

diff --git a/README.md b/README.md
index aa914f03..153fe4c0 100644
--- a/README.md
+++ b/README.md
@@ -55,7 +55,7 @@ Custom Secret Scanning Patterns repository.
 
 
 
-- Generic Passwords
+- Generic Passwords (fewer FPs)
 
 - Generic Password with hex encoded secrets
 
diff --git a/configs/README.md b/configs/README.md
index 4c41bb7a..c2e372dc 100644
--- a/configs/README.md
+++ b/configs/README.md
@@ -120,11 +120,13 @@ _version: v0.1_
 **⚠️ WARNING: THIS RULE IS EXPERIMENTAL AND MIGHT CAUSE A HIGH FALSE POSITIVE RATE (test before commiting to org level) ⚠️**
 Pattern to find hardcoded passwords in YAML configuration files
 
-_version: v0.1_
+_version: v0.2_
 
 **Comments / Notes:**
 
 
+- Expect large numbers of false positives on variables containing 'key' or 'token'
+
 - The hardcoded password is any length
 
 - Some false positives in code might appear
@@ -846,7 +848,9 @@ _version: v0.1_
 
 - The hex token must be 32, 40 or 64 characters long, and contain numbers and letters
 
-- Some false positives in code might appear
+- This may match in code, such as Python, that resembles JSON
+
+- This will not match some isolated fragments of JSON, so be aware of that when testing it
   
 
 <details>
diff --git a/generic/README.md b/generic/README.md
index 6633c3a6..c75181cb 100644
--- a/generic/README.md
+++ b/generic/README.md
@@ -7,13 +7,15 @@
 
 ## Generic Passwords
 
-
+**⚠️ WARNING: THIS RULE IS EXPERIMENTAL AND MIGHT CAUSE A HIGH FALSE POSITIVE RATE (test before commiting to org level) ⚠️**
 
 _version: v0.5_
 
 **Comments / Notes:**
 
 
+- Likely to cause large numbers of false positives - use with caution
+
 - `password`, `secret`, `key`, or password like prefix (fuzzy)
 
 - Delimiters like `=` or `:` (with padding)
@@ -36,7 +38,7 @@ _version: v0.5_
 <summary>Start Pattern</summary>
 
 ```regex
-(?:\A|[^a-zA-Z0-9])(?i)(?:api|auth[a-z]+|jwt|mysql|db)?[_.-]?(?:pass?(?:wo?r?d|code|phrase)|secret|key|token)([_-][a-z0-9]+){0,3}([ \t]+As[ \t]+String)?[\t ]*(={1,3}|:)[\t ]*(?:["']|b["'])?
+(?:\A|[^a-zA-Z0-9])(?i)[a-z0-9_.-]*(?:api|auth[a-z]+|jwt|mysql|db)?[_.-]?(?:pass?(?:wo?r?d|code|phrase)|secret|key|token)([_-][a-z0-9]+){0,3}([ \t]+As[ \t]+String)?[\t ]*(={1,3}|:)[\t ]*(?:["']|b["'])?
 ```
 
 </details><details>
@@ -82,6 +84,80 @@ Add these additional matches to the [Secret Scanning Custom Pattern](https://doc
 
 </details>
 
+## Generic Passwords (fewer FPs)
+
+
+
+_version: v0.1_
+
+**Comments / Notes:**
+
+
+- Expect many false positives if run against vendored-in code, such as JS libraries - use with caution
+
+- `password`, `secret`, `key`, `token` etc. password-like prefix (fuzzy)
+
+- Delimiters like `=` or `:` (with padding)
+
+- Matches fewer characters (A-Za-z0-9_/+.!-), and requires matching quotes around the string
+
+- Attempts to remove variables, placeholders, and common configuration constants such as 'read' and 'write'
+  
+
+<details>
+<summary>Pattern Format</summary>
+
+```regex
+((?i)[a-z0-9_.-]*(api|auth[a-z]*|jwt|mysql|db)[_.-]?)?((?i)pass?(wo?r?d|code|phrase)|secret|token|key)([_-][A-Za-z0-9]+){0,4}_{0,2}(["'`]|[ \t]+As[ \t]+String)?[\t ]*(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)[\t ]*([br]?"[A-Za-z0-9_/+.!-]+"|[br]?'[A-Za-z0-9_/+.!-]+')
+```
+
+</details>
+
+<details>
+<summary>Start Pattern</summary>
+
+```regex
+\A|[^0-9A-Za-z]
+```
+
+</details><details>
+<summary>End Pattern</summary>
+
+```regex
+\z|[^A-Za-z0-9]
+```
+
+</details>
+
+<details>
+<summary>Additional Matches</summary>
+
+Add these additional matches to the [Secret Scanning Custom Pattern](https://docs.github.com/en/enterprise-cloud@latest/code-security/secret-scanning/defining-custom-patterns-for-secret-scanning#example-of-a-custom-pattern-specified-using-additional-requirements).
+
+
+- Not Match:
+
+  ```regex
+  ^[A-Za-z0-9_.-]*(key|KEY|[Tt]oken|TOKEN)(_[a-zA-Z]+)?['"]?\s*([:=]|=>)\s*["']([Ee]mploye[er]|[Ss]taff|([Ss]earch)?[Rr]esult|[a-z][a-zA-Z]+CSX[A-Z][A-Za-z]+|[A-Za-z]*[Bb]ase(64|32|58)|object|claret|assigns?|clean|contains|error|expand|generate|hoist|indent(ation)?|invert|jumps?|pairs?|param(eter)?s?|pop|rewrite|temp(orary)?|token(s|i[sz]e)?|type|((un)?(quote|shift|wrap|finished))|[a-z]{2,10}([A-Z][a-z]{1,15}){1,6}|(compile|is|has|make|add|each|check|close|cache|format|tag|get|set)([A-Z][A-Za-z]+)?|gadget|classic|(try_)?(base|mode|grade|model)|words|identifier|[a-z.-]+\.(jpe?g|(x|ht)ml|txt|docx?|xlsx?|pdf|png)|enabled|name\.invalidPattern|\.|\.data-api|expect|file|config|ansi|Default(Type)?|Cache-Control|((notD|d)eepE|e)qual|name|NAME|package|version|VERSION|start|end|step|async|Event|throws|ok|notOK|verbose|push(Result)?|slimAssertions|(p|notP)ropEqual|((notS|s)trict|not)Equal|value|prev|next|year|key[0-9]?|destroy|[a-z]+EventListeners|timeout|str(ing)?|hmac|uuid|update|find|true|false|[Nn][ui]ll|[Nn]one|[a-z_]+\.((tf)?state|id|key)|(hibernate|ws|err|i18n|employee|bs|org|com|sun|java)(\.[a-zA-Z0-9_]+){1,4}|[A-Z_]+_KEY)["']$
+  ```
+- Not Match:
+
+  ```regex
+  (?i)(token|key)[_-](name|format|type|enabled|success|type|method)\b
+  ```
+- Not Match:
+
+  ```regex
+  ^token(_[A-Z]+)?['"]?\s*[:=]\s*['"](barline|parenthesis|qualified|suport|symbol|statementEnd|singleLineTitle|character|pageBreak|operator|optionalTitle|option|zupfnoter|chordname|macro|error|escape|indent|term|titleUnderline|tag|link|literal|(other|table)Block|list|value|control|set|support|injections|array|doc|source|heading|tokens|storage|empty|newline|empty_line|keyword|comment|meta|[lr]?paren|class|punctuation|regexp?|constant|string|entity|invalid|support|variable|multiline|language|paren|markup|singleline|nospell|text|array|doc|source|heading|tokens)(\.{1,2}[A-Za-z0-9_-]+){0,6}[!.]?["']$
+  ```
+- Not Match:
+
+  ```regex
+  ^KEY_[A-Z]+[0-9]{0,3}: 'k[a-zA-Z0-9]{1,6}'$
+  ```
+
+</details>
+
 ## Generic Password with hex encoded secrets
 
 
@@ -95,7 +171,7 @@ _version: v0.1_
 
 - Delimiters like `=` or `:` (with padding)
 
-- Has to be a token-like value
+- Has to be a token-like value - a 32, 40 or 64 character hex string
   
 
 <details>
@@ -111,7 +187,7 @@ _version: v0.1_
 <summary>Start Pattern</summary>
 
 ```regex
-(?:\A|[^a-zA-Z0-9])(?i)(?:api|auth[a-z]+|jwt|mysql|db)?[_.-]?(?:pass?(?:wo?r?d|code|phrase)|secret|key|token)([_-][a-z0-9]+){0,3}([ \t]+As[ \t]+String)?[\t ]*(={1,3}|:)[\t ]*(?:["']|b["'])?
+(?:\A|[^a-zA-Z0-9])(?i)[a-z0-9._-]*(?:api|auth[a-z]+|jwt|mysql|db)?[_.-]?(?:pass?(?:wo?r?d|code|phrase)|secret|key|token)([_-][a-z0-9]+){0,3}([ \t]+As[ \t]+String)?[\t ]*(={1,3}|:)[\t ]*(?:["']|b["'])?
 ```
 
 </details><details>
@@ -152,7 +228,7 @@ _version: v0.1_
 <summary>Start Pattern</summary>
 
 ```regex
-(?:\A|[^a-zA-Z0-9])(?i)(?:api|auth[a-z]+|jwt|mysql|db)?[_.-]?(?:pass?(?:wo?r?d|code|phrase)|secret|key|token)([_-][a-z0-9]+){0,3}([ \t]+As[ \t]+String)?[\t ]*(={1,3}|:)[\t ]*(?:["']|b["'])?
+(?:\A|[^a-zA-Z0-9])(?i)[a-z0-9._-]*(?:api|auth[a-z]+|jwt|mysql|db)?[_.-]?(?:pass?(?:wo?r?d|code|phrase)|secret|key|token)([_-][a-z0-9]+){0,3}([ \t]+As[ \t]+String)?[\t ]*(={1,3}|:)[\t ]*(?:["']|b["'])?
 ```
 
 </details><details>
@@ -286,7 +362,7 @@ _version: v0.2_
 <summary>Start Pattern</summary>
 
 ```regex
-\b([Bb]earer |[Tt]oken (token=)?)
+(Authorization: |['"])([Bb]earer |[Tt]oken (token=)?)
 ```
 
 </details><details>

From f9560fc4a690e5d65c96f18f9a98d727be46de55 Mon Sep 17 00:00:00 2001
From: aegilops <41705651+aegilops@users.noreply.github.com>
Date: Fri, 24 May 2024 16:41:47 +0100
Subject: [PATCH 57/95] Updated PagerDuty pattern to remove , as separator

---
 vendors/patterns.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/vendors/patterns.yml b/vendors/patterns.yml
index 1e0d723f..ff51c92b 100644
--- a/vendors/patterns.yml
+++ b/vendors/patterns.yml
@@ -363,7 +363,7 @@ patterns:
       pattern: |
         [A-Za-z0-9_-]{20}
       start: |
-        (\A|\b)(?i)((pd|pagerduty)_(service|api)_key['"`]?\s*([:=]|[?:]=|[=-]>|,)\s*['"`]?|Token token=)
+        (\A|\b)(?i)((pd|pagerduty)_(service|api)_key['"`]?\s*([:=]|[?:]=|[=-]>)\s*['"`]?|Token token=)
       end: |
         \z|\b
       additional_match:

From 9385ceb19c3f01ddcbe1a964f7c39e90651c3e55 Mon Sep 17 00:00:00 2001
From: aegilops <41705651+aegilops@users.noreply.github.com>
Date: Fri, 24 May 2024 16:42:26 +0100
Subject: [PATCH 58/95] Updated README.md

---
 configs/README.md | 35 +++++++++++++++++++++++++----------
 vendors/README.md |  2 +-
 2 files changed, 26 insertions(+), 11 deletions(-)

diff --git a/configs/README.md b/configs/README.md
index c2e372dc..963a2d04 100644
--- a/configs/README.md
+++ b/configs/README.md
@@ -401,11 +401,9 @@ _version: v0.1_
 
 - The hardcoded password is any length
 
-- Some false positives in code might appear
-
-- The pattern only checks for certain key words to end the variable name (`secret`, `password`, etc.)
+- Some false positives in code or YAML files might appear, especially where the variable is called 'key' or 'token'
 
-- This will catch the start of a multiline password, but the end will not be found if it is on a different line
+- The pattern checks for certain key words to end the variable name (`secret`, `password`, etc.)
   
 
 <details>
@@ -567,7 +565,7 @@ _version: v0.1_
 ## .env file style secrets
 
 **⚠️ WARNING: THIS RULE IS EXPERIMENTAL AND MIGHT CAUSE A HIGH FALSE POSITIVE RATE (test before commiting to org level) ⚠️**
-Find .env file style secrets in configuration files
+Find .env file style secrets in configuration files such as .env, Dockerfile, shell scripts etc.
 
 _version: v0.1_
 
@@ -576,7 +574,9 @@ _version: v0.1_
 
 - Looks for secrets in the format of `SECRET=secret` at the start of a line, possibly with an `ENV ` or `export ` prefix
 
-- Some false positives in code might appear
+- Allows no whitespace in the secret, to cut false positives
+
+- Some false positives in code might appear, especially where the variable name is 'key' or 'token'
 
 - The pattern only checks for certain key words to begin the pattern (`secret`, `password`, etc.)
 
@@ -596,14 +596,14 @@ _version: v0.1_
 <summary>Start Pattern</summary>
 
 ```regex
-(?:\n|\A)(ENV |export )?[A-Z_]*(?:SECRET|SERVICE_PASS(WD|WORD|CODE|PHRASE)|PASS(?:WD|WORD|CODE|PHRASE)?|KEY)=['"]?
+(?:\n|\A)((export|ENV|ARG) )?[A-Z_]*(?:SECRET|PASS(?:WD|WOR[TD]|CODE|PHRASE)?|KEY|TOKEN)=['"]?
 ```
 
 </details><details>
 <summary>End Pattern</summary>
 
 ```regex
-['"\r\n#]|\z
+[\r\n#]|['"]\s*[\r\n]|\z
 ```
 
 </details>
@@ -617,12 +617,27 @@ Add these additional matches to the [Secret Scanning Custom Pattern](https://doc
 - Not Match:
 
   ```regex
-  ^\$[{(]
+  ^\$(\{[^}]+\}|\([^)]+)|[A-Za-z_]+|[0-9])$
+  ```
+- Not Match:
+
+  ```regex
+  ^(<[^>]+>|\[[^]+\]|\{[^}+\}|(your|my|the|a)_[a-z_]+|.*(passwor[t]|key|secret|token|密码).*|\.\.\.|xxx+|yyy+|zzz+|aaa+|bbb+|ccc+)$
+  ```
+- Not Match:
+
+  ```regex
+  ^(test|value)([._-][a-z_.-]+)?$
+  ```
+- Not Match:
+
+  ```regex
+  ^(?i)(true|false|y(es)?|no?|on|off|0|1|nill|null|none|(\\x[a-f0-9]{2})+)$
   ```
 - Not Match:
 
   ```regex
-  ^<[^>]+>$
+  ^(/|file:///|[A-Za-z]:/)[A-Za-z0-9._-]{3,}+(/[a-z._-]{1,}){2,}/?$
   ```
 
 </details>
diff --git a/vendors/README.md b/vendors/README.md
index 953ea734..0b0efd65 100644
--- a/vendors/README.md
+++ b/vendors/README.md
@@ -660,7 +660,7 @@ _version: v0.1_
 <summary>Start Pattern</summary>
 
 ```regex
-(\A|\b)(?i)((pd|pagerduty)_(service|api)_key['"`]?\s*([:=]|[?:]=|[=-]>|,)\s*['"`]?|Token token=)
+(\A|\b)(?i)((pd|pagerduty)_(service|api)_key['"`]?\s*([:=]|[?:]=|[=-]>)\s*['"`]?|Token token=)
 ```
 
 </details><details>

From 11e61022e3e71e6ed37eb082224b8ef6ddd511cb Mon Sep 17 00:00:00 2001
From: aegilops <41705651+aegilops@users.noreply.github.com>
Date: Fri, 24 May 2024 16:48:42 +0100
Subject: [PATCH 59/95] Updated Sentry vendor patterns

---
 vendors/patterns.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/vendors/patterns.yml b/vendors/patterns.yml
index ff51c92b..5c16a953 100644
--- a/vendors/patterns.yml
+++ b/vendors/patterns.yml
@@ -58,7 +58,7 @@ patterns:
       pattern: |
         [a-fA-F0-9]{64}
       start: |
-        (?:\A|[\r\n])(?:\[auth\][^[]*\ntoken\s*=|(?:export )?SENTRY_AUTH_TOKEN\s*=|sentry-cli [^\r\n]*--auth-token |auth\.token\s*=)\s*['"`]?
+        (?:(?:\A|[\r\n])\[auth\][^[]*\ntoken\s*=|(?:\A|\b)SENTRY_AUTH_TOKEN\s*=|(?:\A|\b)sentry-cli [^\r\n]*--auth-token |(?:\A|\b)auth\.token\s*=)\s*['"`]?
       end: |
         \z|\s|['"`]
       additional_match:
@@ -88,7 +88,7 @@ patterns:
       pattern: |
         [a-fA-F0-9]{32}
       start: |
-        (?:\A|[\r\n])(?:\[auth\][^[]*\napi_key\s*=|(?:export )?SENTRY_API_KEY\s*=|sentry-cli [^\r\n]*--api-key |auth\.api_key\s*=)\s*['"`]?
+        (?:(?:\A|[\r\n])\[auth\][^[]*\napi_key\s*=|(?:\A|\b)SENTRY_API_KEY\s*=|(?:\A|\b)sentry-cli [^\r\n]*--api-key |(?:\A|\b)auth\.api_key\s*=)\s*['"`]?
       end: |
         \z|\s|['"`]
       additional_match:

From be235e2e978bff57fc3698ca8768607b12f95ab5 Mon Sep 17 00:00:00 2001
From: aegilops <41705651+aegilops@users.noreply.github.com>
Date: Fri, 24 May 2024 16:48:46 +0100
Subject: [PATCH 60/95] Updated README.md

---
 vendors/README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/vendors/README.md b/vendors/README.md
index 0b0efd65..50e0da16 100644
--- a/vendors/README.md
+++ b/vendors/README.md
@@ -98,7 +98,7 @@ _version: v0.1_
 <summary>Start Pattern</summary>
 
 ```regex
-(?:\A|[\r\n])(?:\[auth\][^[]*\ntoken\s*=|(?:export )?SENTRY_AUTH_TOKEN\s*=|sentry-cli [^\r\n]*--auth-token |auth\.token\s*=)\s*['"`]?
+(?:(?:\A|[\r\n])\[auth\][^[]*\ntoken\s*=|(?:\A|\b)SENTRY_AUTH_TOKEN\s*=|(?:\A|\b)sentry-cli [^\r\n]*--auth-token |(?:\A|\b)auth\.token\s*=)\s*['"`]?
 ```
 
 </details><details>
@@ -146,7 +146,7 @@ _version: v0.1_
 <summary>Start Pattern</summary>
 
 ```regex
-(?:\A|[\r\n])(?:\[auth\][^[]*\napi_key\s*=|(?:export )?SENTRY_API_KEY\s*=|sentry-cli [^\r\n]*--api-key |auth\.api_key\s*=)\s*['"`]?
+(?:(?:\A|[\r\n])\[auth\][^[]*\napi_key\s*=|(?:\A|\b)SENTRY_API_KEY\s*=|(?:\A|\b)sentry-cli [^\r\n]*--api-key |(?:\A|\b)auth\.api_key\s*=)\s*['"`]?
 ```
 
 </details><details>

From 806e3fe3bc60830a364eabd81e3f2316335810dd Mon Sep 17 00:00:00 2001
From: aegilops <41705651+aegilops@users.noreply.github.com>
Date: Fri, 24 May 2024 16:53:41 +0100
Subject: [PATCH 61/95] Removing beep:boop common sample Base64 bearer token

---
 generic/patterns.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/generic/patterns.yml b/generic/patterns.yml
index 82b394f3..67e6e118 100644
--- a/generic/patterns.yml
+++ b/generic/patterns.yml
@@ -165,7 +165,7 @@ patterns:
       end: |
         \z|[\s'"]
       additional_not_match:
-        - ^(?:letmein|Oracle|SuperSecretString|foo|ababbdbbebbbebdbbe5538003023|XYZ_INVALID_ACCESTOKEN_XYZ|QQ==|Shizuku|mF_9.B5f-4.1JqM|h480djs93hd8|SlAV32hkKG)$
+        - ^(?:letmein|Oracle|SuperSecretString|foo|ababbdbbebbbebdbbe5538003023|XYZ_INVALID_ACCESTOKEN_XYZ|QQ==|Shizuku|mF_9.B5f-4.1JqM|h480djs93hd8|SlAV32hkKG|YmVlcDpib29w)$
         - ^(?i)(?:dummy|fake|bearer|auth|invalid|your|my|the|undefined|github|oidc|database)(?:_api)?(?:_?token|key|secret)?$
         - ^(?i)(?:[a-z0-9]|XYZ|ABC|123|.*_token)$
         - (?i)x{5}

From 553d92515bef5c882ec4e3e383d8b54b3bd47ad0 Mon Sep 17 00:00:00 2001
From: aegilops <41705651+aegilops@users.noreply.github.com>
Date: Fri, 24 May 2024 16:53:46 +0100
Subject: [PATCH 62/95] Updated README.md

---
 generic/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/generic/README.md b/generic/README.md
index c75181cb..9d1cba76 100644
--- a/generic/README.md
+++ b/generic/README.md
@@ -383,7 +383,7 @@ Add these additional matches to the [Secret Scanning Custom Pattern](https://doc
 - Not Match:
 
   ```regex
-  ^(?:letmein|Oracle|SuperSecretString|foo|ababbdbbebbbebdbbe5538003023|XYZ_INVALID_ACCESTOKEN_XYZ|QQ==|Shizuku|mF_9.B5f-4.1JqM|h480djs93hd8|SlAV32hkKG)$
+  ^(?:letmein|Oracle|SuperSecretString|foo|ababbdbbebbbebdbbe5538003023|XYZ_INVALID_ACCESTOKEN_XYZ|QQ==|Shizuku|mF_9.B5f-4.1JqM|h480djs93hd8|SlAV32hkKG|YmVlcDpib29w)$
   ```
 - Not Match:
 

From 7e39a0854f090f2e0d2d5eab118c0a3f8adb5770 Mon Sep 17 00:00:00 2001
From: aegilops <41705651+aegilops@users.noreply.github.com>
Date: Tue, 11 Jun 2024 08:44:41 +0100
Subject: [PATCH 63/95] Fix comment

---
 configs/patterns.yml | 26 +++++++++++++++++---------
 1 file changed, 17 insertions(+), 9 deletions(-)

diff --git a/configs/patterns.yml b/configs/patterns.yml
index bbf71dd2..484c0f29 100644
--- a/configs/patterns.yml
+++ b/configs/patterns.yml
@@ -220,9 +220,8 @@ patterns:
         \n\n|\r\n\r\n|(\n|\r\n)[ \t]+\S+:|(\n|\r\n)\.\.\.[ \t\n\r]|\z
     comments:
       - "The hardcoded password is any length"
-      - "Some false positives in code might appear"
-      - "The pattern only checks for certain key words to end the variable name (`secret`, `password`, etc.)"
-      - "This will catch the start of a multiline password, but the end will not be found if it is on a different line"
+      - "Some false positives in code or YAML files might appear, especially where the variable is called 'key' or 'token'"
+      - "The pattern checks for certain key words to end the variable name (`secret`, `password`, etc.)"
     test:
       data: |
         password: |
@@ -308,22 +307,31 @@ patterns:
   - name: .env file style secrets
     type: env_passwords
     description: |
-      Find .env file style secrets in configuration files
+      Find .env file style secrets in configuration files such as .env, Dockerfile, shell scripts etc.
     experimental: true
     regex:
       version: 0.1
       pattern: |
         [^\r\n\x00-\x08'"#]+
       start: |
-        (?:\n|\A)(ENV |export )?[A-Z_]*(?:SECRET|SERVICE_PASS(WD|WORD|CODE|PHRASE)|PASS(?:WD|WORD|CODE|PHRASE)?|KEY)=['"]?
+        (?:\n|\A)((export|ENV|ARG) )?[A-Z_]*(?:SECRET|PASS(?:WD|WOR[TD]|CODE|PHRASE)?|KEY|TOKEN)=['"]?
       end: |
-        ['"\r\n#]|\z
+        [\r\n#]|['"]\s*[\r\n]|\z
       additional_not_match:
-        - ^\$[{(]
-        - ^<[^>]+>$
+        # variable, process substitution
+        - ^\$(\{[^}]+\}|\([^)]+)|[A-Za-z_]+|[0-9])$
+        # placeholders
+        - ^(<[^>]+>|\[[^]+\]|\{[^}+\}|(your|my|the|a)_[a-z_]+|.*(passwor[t]|key|secret|token|密码).*|\.\.\.|xxx+|yyy+|zzz+|aaa+|bbb+|ccc+)$
+        # non-secret related "key" content
+        - ^(test|value)([._-][a-z_.-]+)?$
+        # non-secret related content
+        - ^(?i)(true|false|y(es)?|no?|on|off|0|1|nill|null|none|(\\x[a-f0-9]{2})+)$
+        # a path
+        - '^(/|file:///|[A-Za-z]:/)[A-Za-z0-9._-]{3,}+(/[a-z._-]{1,}){2,}/?$'
     comments:
       - "Looks for secrets in the format of `SECRET=secret` at the start of a line, possibly with an `ENV ` or `export ` prefix"
-      - "Some false positives in code might appear"
+      - "Allows no whitespace in the secret, to cut false positives"
+      - "Some false positives in code might appear, especially where the variable name is 'key' or 'token'"
       - "The pattern only checks for certain key words to begin the pattern (`secret`, `password`, etc.)"
       - "More restrictive than the Generic Passwords pattern, so less prone to false positives"
     test:

From 0f334c46230a329bebd9c9de5bbe5b7d2fecd7d9 Mon Sep 17 00:00:00 2001
From: aegilops <41705651+aegilops@users.noreply.github.com>
Date: Wed, 10 Jul 2024 14:02:55 +0100
Subject: [PATCH 64/95] Updated Base64 pattern to be accurate and support URI
 safe version of Base64

---
 configs/patterns.yml | 62 ++++++++++++++++++++++++++++++++++++++++++--
 generic/patterns.yml | 28 +++++++++++++++++++-
 2 files changed, 87 insertions(+), 3 deletions(-)

diff --git a/configs/patterns.yml b/configs/patterns.yml
index 484c0f29..b574b240 100644
--- a/configs/patterns.yml
+++ b/configs/patterns.yml
@@ -349,7 +349,7 @@ patterns:
     regex:
       version: 0.1
       pattern: |
-        (([A-Za-z0-9+/]){4})+[A-Za-z0-9+/]{1,2}={0,2}
+        (([A-Za-z0-9+/]){4})*([A-Za-z0-9+/]{4}|[A-Za-z0-9+/]{3}=|[A-Za-z0-9+/]{2}==)
       start: |
         (?:\n|\A)[ \t]*(?i)[a-z_-]*(?:secret|service_pass(wd|word|code|phrase)|pass(?:wd|word|code|phrase)?|key|token)[ \t]*:[ \t]*(['"]?|[|>]-?[ \t]*\n[ \t]*)
       end: |
@@ -368,6 +368,34 @@ patterns:
       end_offset: 26
 
 
+  - name: YAML with URI-safe Base64 encoded secrets
+    type: yaml_base64_uri
+    description: |
+      Hardcoded URI-safe Base64-encoded passwords in YAML configuration files
+    experimental: true
+    regex:
+      version: 0.1
+      pattern: |
+        (([A-Za-z0-9_-]){4})*([A-Za-z0-9_-]{4}|[A-Za-z0-9_-]{3}=|[A-Za-z0-9_-]{2}==)
+      start: |
+        (?:\n|\A)[ \t]*(?i)[a-z_-]*(?:secret|service_pass(wd|word|code|phrase)|pass(?:wd|word|code|phrase)?|key|token)[ \t]*:[ \t]*(['"]?|[|>]-?[ \t]*\n[ \t]*)
+      end: |
+        ['"\r\n]|\z
+      additional_match:
+        - '[0-9]'
+        - '[A-Z]'
+        - '[a-z]'
+        - '^.{12,}$'
+    comments:
+      - "The Base64 must contain numbers, upper case and lower case and be at least 12 characters long"
+      - "Some false positives in code might appear"
+      - "This matches _- instead of +/, for URI-safe Base64"
+    test:
+      data: "password: AAAAAAAAAAAa00=="
+      start_offset: 10
+      end_offset: 26
+
+
   - name: YAML with hex token
     type: yaml_token
     description: |
@@ -401,7 +429,36 @@ patterns:
     regex:
       version: 0.1
       pattern: |
-        (([A-Za-z0-9+/]){4})+[A-Za-z0-9+/]{1,2}={0,2}
+        (([A-Za-z0-9+/]){4})*([A-Za-z0-9+/]{4}|[A-Za-z0-9+/]{3}=|[A-Za-z0-9+/]{2}==)
+      start: |
+        [{[,][ \t]*[ \t\r\n]*"(?i)[a-z_.-]*(?:secret|service_pass(wd|word|code|phrase)|pass(?:wd|word|code|phrase)?|key|token)"[ \t]*:[ \t]*"
+      end: |
+        "[ \t\r\n]*[,}\]]
+      additional_match:
+        - '[0-9]'
+        - '[A-Z]'
+        - '[a-z]'
+        - '^.{12,}$'
+    comments:
+      - "The Base64 must contain numbers, upper case and lower case and be at least 12 characters long"
+      - "This may match in code, such as Python, that resembles JSON"
+      - "This will not match some isolated fragments of JSON, so be aware of that when testing it"
+    test:
+      data: |
+        {"password": "AAAAAAAAAAAa00==", "user": "admin"}
+      start_offset: 14
+      end_offset: 30
+
+
+  - name: JSON with URI-safe Base64 encoded secrets
+    type: json_base64_uri
+    description: |
+      Hardcoded URI-safe Base64-encoded passwords in JSON configuration files
+    experimental: true
+    regex:
+      version: 0.1
+      pattern: |
+        (([A-Za-z0-9_-]){4})*([A-Za-z0-9_-]{4}|[A-Za-z0-9_-]{3}=|[A-Za-z0-9_-]{2}==)
       start: |
         [{[,][ \t]*[ \t\r\n]*"(?i)[a-z_.-]*(?:secret|service_pass(wd|word|code|phrase)|pass(?:wd|word|code|phrase)?|key|token)"[ \t]*:[ \t]*"
       end: |
@@ -415,6 +472,7 @@ patterns:
       - "The Base64 must contain numbers, upper case and lower case and be at least 12 characters long"
       - "This may match in code, such as Python, that resembles JSON"
       - "This will not match some isolated fragments of JSON, so be aware of that when testing it"
+      - "This matches _- instead of +/, for URI-safe Base64"
     test:
       data: |
         {"password": "AAAAAAAAAAAa00==", "user": "admin"}
diff --git a/generic/patterns.yml b/generic/patterns.yml
index 67e6e118..4e37004d 100644
--- a/generic/patterns.yml
+++ b/generic/patterns.yml
@@ -103,7 +103,7 @@ patterns:
     regex:
       version: 0.1
       pattern: |
-        (([A-Za-z0-9+/]){4})+[A-Za-z0-9+/]{1,2}={0,2}
+        (([A-Za-z0-9+/]){4})*([A-Za-z0-9+/]{4}|[A-Za-z0-9+/]{3}=|[A-Za-z0-9+/]{2}==)
       start: |
         (?:\A|[^a-zA-Z0-9])(?i)[a-z0-9._-]*(?:api|auth[a-z]+|jwt|mysql|db)?[_.-]?(?:pass?(?:wo?r?d|code|phrase)|secret|key|token)([_-][a-z0-9]+){0,3}([ \t]+As[ \t]+String)?[\t ]*(={1,3}|:)[\t ]*(?:["']|b["'])?
       end: |
@@ -123,6 +123,32 @@ patterns:
       - "Delimiters like `=` or `:` (with padding)"
 
 
+  - name: Generic Password with URI-safe Base64 encoded secrets
+    type: generic_passwords_base64_uri
+    regex:
+      version: 0.1
+      pattern: |
+        (([A-Za-z0-9_-]){4})*([A-Za-z0-9_-]{4}|[A-Za-z0-9_-]{3}=|[A-Za-z0-9_-]{2}==)
+      start: |
+        (?:\A|[^a-zA-Z0-9])(?i)[a-z0-9._-]*(?:api|auth[a-z]+|jwt|mysql|db)?[_.-]?(?:pass?(?:wo?r?d|code|phrase)|secret|key|token)([_-][a-z0-9]+){0,3}([ \t]+As[ \t]+String)?[\t ]*(={1,3}|:)[\t ]*(?:["']|b["'])?
+      end: |
+        (\z|[\r\n'"])
+      additional_match:
+        - '[0-9]'
+        - '[A-Z]'
+        - '[a-z]'
+        - '^.{12,}$'
+    test:
+      data: password="AAAAAAAAAAAa00=="
+      start_offset: 10
+      end_offset: 26
+    comments:
+      - "The Base64 must contain numbers, upper case and lower case and be at least 12 characters long"
+      - "`password`, `secret`, `key`, or password like prefix (fuzzy)"
+      - "Delimiters like `=` or `:` (with padding)"
+      - "This matches _- instead of +/, for URI-safe Base64"
+
+
   - name: UUIDs
     type: uuids
     regex:

From 2ab1333068ba943b0f556ab4d56158a19461b171 Mon Sep 17 00:00:00 2001
From: aegilops <41705651+aegilops@users.noreply.github.com>
Date: Wed, 10 Jul 2024 14:03:33 +0100
Subject: [PATCH 65/95] Updated README.md

---
 README.md         |   2 +
 configs/README.md | 156 +++++++++++++++++++++++++++++++++++++++++++++-
 generic/README.md |  78 ++++++++++++++++++++++-
 3 files changed, 233 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index 153fe4c0..9f2acb09 100644
--- a/README.md
+++ b/README.md
@@ -61,6 +61,8 @@ Custom Secret Scanning Patterns repository.
 
 - Generic Password with Base64 encoded secrets
 
+- Generic Password with URI-safe Base64 encoded secrets
+
 - UUIDs
 
 - Bearer Tokens
diff --git a/configs/README.md b/configs/README.md
index 963a2d04..1d6db976 100644
--- a/configs/README.md
+++ b/configs/README.md
@@ -661,7 +661,82 @@ _version: v0.1_
 <summary>Pattern Format</summary>
 
 ```regex
-(([A-Za-z0-9+/]){4})+[A-Za-z0-9+/]{1,2}={0,2}
+(([A-Za-z0-9+/]){4})*([A-Za-z0-9+/]{4}|[A-Za-z0-9+/]{3}=|[A-Za-z0-9+/]{2}==)
+```
+
+</details>
+
+<details>
+<summary>Start Pattern</summary>
+
+```regex
+(?:\n|\A)[ \t]*(?i)[a-z_-]*(?:secret|service_pass(wd|word|code|phrase)|pass(?:wd|word|code|phrase)?|key|token)[ \t]*:[ \t]*(['"]?|[|>]-?[ \t]*\n[ \t]*)
+```
+
+</details><details>
+<summary>End Pattern</summary>
+
+```regex
+['"\r\n]|\z
+```
+
+</details>
+
+<details>
+<summary>Additional Matches</summary>
+
+Add these additional matches to the [Secret Scanning Custom Pattern](https://docs.github.com/en/enterprise-cloud@latest/code-security/secret-scanning/defining-custom-patterns-for-secret-scanning#example-of-a-custom-pattern-specified-using-additional-requirements).
+
+
+
+- Match:
+
+  ```regex
+  [0-9]
+  ```
+
+- Match:
+
+  ```regex
+  [A-Z]
+  ```
+
+- Match:
+
+  ```regex
+  [a-z]
+  ```
+
+- Match:
+
+  ```regex
+  ^.{12,}$
+  ```
+
+</details>
+
+## YAML with URI-safe Base64 encoded secrets
+
+**⚠️ WARNING: THIS RULE IS EXPERIMENTAL AND MIGHT CAUSE A HIGH FALSE POSITIVE RATE (test before commiting to org level) ⚠️**
+Hardcoded URI-safe Base64-encoded passwords in YAML configuration files
+
+_version: v0.1_
+
+**Comments / Notes:**
+
+
+- The Base64 must contain numbers, upper case and lower case and be at least 12 characters long
+
+- Some false positives in code might appear
+
+- This matches _- instead of +/, for URI-safe Base64
+  
+
+<details>
+<summary>Pattern Format</summary>
+
+```regex
+(([A-Za-z0-9_-]){4})*([A-Za-z0-9_-]{4}|[A-Za-z0-9_-]{3}=|[A-Za-z0-9_-]{2}==)
 ```
 
 </details>
@@ -797,7 +872,84 @@ _version: v0.1_
 <summary>Pattern Format</summary>
 
 ```regex
-(([A-Za-z0-9+/]){4})+[A-Za-z0-9+/]{1,2}={0,2}
+(([A-Za-z0-9+/]){4})*([A-Za-z0-9+/]{4}|[A-Za-z0-9+/]{3}=|[A-Za-z0-9+/]{2}==)
+```
+
+</details>
+
+<details>
+<summary>Start Pattern</summary>
+
+```regex
+[{[,][ \t]*[ \t\r\n]*"(?i)[a-z_.-]*(?:secret|service_pass(wd|word|code|phrase)|pass(?:wd|word|code|phrase)?|key|token)"[ \t]*:[ \t]*"
+```
+
+</details><details>
+<summary>End Pattern</summary>
+
+```regex
+"[ \t\r\n]*[,}\]]
+```
+
+</details>
+
+<details>
+<summary>Additional Matches</summary>
+
+Add these additional matches to the [Secret Scanning Custom Pattern](https://docs.github.com/en/enterprise-cloud@latest/code-security/secret-scanning/defining-custom-patterns-for-secret-scanning#example-of-a-custom-pattern-specified-using-additional-requirements).
+
+
+
+- Match:
+
+  ```regex
+  [0-9]
+  ```
+
+- Match:
+
+  ```regex
+  [A-Z]
+  ```
+
+- Match:
+
+  ```regex
+  [a-z]
+  ```
+
+- Match:
+
+  ```regex
+  ^.{12,}$
+  ```
+
+</details>
+
+## JSON with URI-safe Base64 encoded secrets
+
+**⚠️ WARNING: THIS RULE IS EXPERIMENTAL AND MIGHT CAUSE A HIGH FALSE POSITIVE RATE (test before commiting to org level) ⚠️**
+Hardcoded URI-safe Base64-encoded passwords in JSON configuration files
+
+_version: v0.1_
+
+**Comments / Notes:**
+
+
+- The Base64 must contain numbers, upper case and lower case and be at least 12 characters long
+
+- This may match in code, such as Python, that resembles JSON
+
+- This will not match some isolated fragments of JSON, so be aware of that when testing it
+
+- This matches _- instead of +/, for URI-safe Base64
+  
+
+<details>
+<summary>Pattern Format</summary>
+
+```regex
+(([A-Za-z0-9_-]){4})*([A-Za-z0-9_-]{4}|[A-Za-z0-9_-]{3}=|[A-Za-z0-9_-]{2}==)
 ```
 
 </details>
diff --git a/generic/README.md b/generic/README.md
index 9d1cba76..86e8a194 100644
--- a/generic/README.md
+++ b/generic/README.md
@@ -219,7 +219,83 @@ _version: v0.1_
 <summary>Pattern Format</summary>
 
 ```regex
-(([A-Za-z0-9+/]){4})+[A-Za-z0-9+/]{1,2}={0,2}
+(([A-Za-z0-9+/]){4})*([A-Za-z0-9+/]{4}|[A-Za-z0-9+/]{3}=|[A-Za-z0-9+/]{2}==)
+```
+
+</details>
+
+<details>
+<summary>Start Pattern</summary>
+
+```regex
+(?:\A|[^a-zA-Z0-9])(?i)[a-z0-9._-]*(?:api|auth[a-z]+|jwt|mysql|db)?[_.-]?(?:pass?(?:wo?r?d|code|phrase)|secret|key|token)([_-][a-z0-9]+){0,3}([ \t]+As[ \t]+String)?[\t ]*(={1,3}|:)[\t ]*(?:["']|b["'])?
+```
+
+</details><details>
+<summary>End Pattern</summary>
+
+```regex
+(\z|[\r\n'"])
+```
+
+</details>
+
+<details>
+<summary>Additional Matches</summary>
+
+Add these additional matches to the [Secret Scanning Custom Pattern](https://docs.github.com/en/enterprise-cloud@latest/code-security/secret-scanning/defining-custom-patterns-for-secret-scanning#example-of-a-custom-pattern-specified-using-additional-requirements).
+
+
+
+- Match:
+
+  ```regex
+  [0-9]
+  ```
+
+- Match:
+
+  ```regex
+  [A-Z]
+  ```
+
+- Match:
+
+  ```regex
+  [a-z]
+  ```
+
+- Match:
+
+  ```regex
+  ^.{12,}$
+  ```
+
+</details>
+
+## Generic Password with URI-safe Base64 encoded secrets
+
+
+
+_version: v0.1_
+
+**Comments / Notes:**
+
+
+- The Base64 must contain numbers, upper case and lower case and be at least 12 characters long
+
+- `password`, `secret`, `key`, or password like prefix (fuzzy)
+
+- Delimiters like `=` or `:` (with padding)
+
+- This matches _- instead of +/, for URI-safe Base64
+  
+
+<details>
+<summary>Pattern Format</summary>
+
+```regex
+(([A-Za-z0-9_-]){4})*([A-Za-z0-9_-]{4}|[A-Za-z0-9_-]{3}=|[A-Za-z0-9_-]{2}==)
 ```
 
 </details>

From c0a165621e17d4048f6be749949dbd4cb122b8f9 Mon Sep 17 00:00:00 2001
From: aegilops <41705651+aegilops@users.noreply.github.com>
Date: Wed, 10 Jul 2024 14:07:47 +0100
Subject: [PATCH 66/95] Script to help update markdown using
 secret-scanning-tools helper script

---
 update_markdown.sh | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100755 update_markdown.sh

diff --git a/update_markdown.sh b/update_markdown.sh
new file mode 100755
index 00000000..b94659cf
--- /dev/null
+++ b/update_markdown.sh
@@ -0,0 +1,8 @@
+#!/bin/bash
+
+if [ -z "$SECRET_SCANNING_TOOLS_PATH" ]; then
+  export SECRET_SCANNING_TOOLS_PATH="${HOME}"/secret-scanning-tools
+  echo "Defaulting to SECRET_SCANNING_TOOLS_PATH=${SECRET_SCANNING_TOOLS_PATH}"
+fi
+
+CUSTOM_PATTERNS_PATH=$PWD "${SECRET_SCANNING_TOOLS_PATH}"/examples/update_custom_patterns_readme.sh

From 65bf0e73146356116fcbfb6683169b642e322ed9 Mon Sep 17 00:00:00 2001
From: aegilops <41705651+aegilops@users.noreply.github.com>
Date: Wed, 10 Jul 2024 14:25:28 +0100
Subject: [PATCH 67/95] Fixing NOT match regex

---
 generic/patterns.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/generic/patterns.yml b/generic/patterns.yml
index 4e37004d..e43cf3b5 100644
--- a/generic/patterns.yml
+++ b/generic/patterns.yml
@@ -15,7 +15,7 @@ patterns:
       additional_not_match:
         # something that means "password" or a placeholder name - either a variable or a placeholder
         # a literal value or config switch
-        - _?)?(?:[a-zA-Z0-9._]+[_.])?(?:the )?(?:pass?(wo?r?d|code|phrase)|pass|pwd|secret|token|tok|redacted|placeholder|dummy|pw|thephrase)|write|read|on|off|true|false|none|null( \? )?|nil|undefined|eof|ignore|eol|git|yes|no|y|n),?\s*\){0,2}[\]>)]?(?:\)\s*\{)?\\?(( or | \|\| ).*)?$
+        - _?\)?(?:[a-zA-Z0-9._]+[_.])?(?:the )?(?:pass?(wo?r?d|code|phrase)|pass|pwd|secret|token|tok|redacted|placeholder|dummy|pw|thephrase)|write|read|on|off|true|false|none|null( \? )?|nil|undefined|eof|ignore|eol|git|yes|no|y|n),?\s*\){0,2}[\]>)]?(?:\)\s*\{)?\\?(( or | \|\| ).*)?$
         # Python type hints, Swift typing
         - ^\s*(?:(?:typing\.)?(?:(?:[Tt]uple|[Ll]ist|[Dd]ict|Callable|Iterable|Sequence|Optional|Union)\[.*|(?:int|str|float|(?:typing.)?Any|None|bytes|bool|ReadableBuffer)\s*(?:[,|].*)?|(?:Int|Swift\.Int|Int32)\.*))\s*$
         # ..., \, , \n, \0, ',' and other single chars, smilies, hex, digits, nothing at all,

From 90d5aa9e04773cfa2516467924ffcf128e62b02d Mon Sep 17 00:00:00 2001
From: aegilops <41705651+aegilops@users.noreply.github.com>
Date: Wed, 10 Jul 2024 14:25:43 +0100
Subject: [PATCH 68/95] Updated README.md

---
 generic/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/generic/README.md b/generic/README.md
index 86e8a194..e1916917 100644
--- a/generic/README.md
+++ b/generic/README.md
@@ -59,7 +59,7 @@ Add these additional matches to the [Secret Scanning Custom Pattern](https://doc
 - Not Match:
 
   ```regex
-  _?)?(?:[a-zA-Z0-9._]+[_.])?(?:the )?(?:pass?(wo?r?d|code|phrase)|pass|pwd|secret|token|tok|redacted|placeholder|dummy|pw|thephrase)|write|read|on|off|true|false|none|null( \? )?|nil|undefined|eof|ignore|eol|git|yes|no|y|n),?\s*\){0,2}[\]>)]?(?:\)\s*\{)?\\?(( or | \|\| ).*)?$
+  _?\)?(?:[a-zA-Z0-9._]+[_.])?(?:the )?(?:pass?(wo?r?d|code|phrase)|pass|pwd|secret|token|tok|redacted|placeholder|dummy|pw|thephrase)|write|read|on|off|true|false|none|null( \? )?|nil|undefined|eof|ignore|eol|git|yes|no|y|n),?\s*\){0,2}[\]>)]?(?:\)\s*\{)?\\?(( or | \|\| ).*)?$
   ```
 - Not Match:
 

From fc399a066e0d4a2da37d5155e964da35474ce024 Mon Sep 17 00:00:00 2001
From: aegilops <41705651+aegilops@users.noreply.github.com>
Date: Wed, 10 Jul 2024 14:28:35 +0100
Subject: [PATCH 69/95] Fix errant bracket

---
 generic/patterns.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/generic/patterns.yml b/generic/patterns.yml
index e43cf3b5..dc1143d6 100644
--- a/generic/patterns.yml
+++ b/generic/patterns.yml
@@ -15,7 +15,7 @@ patterns:
       additional_not_match:
         # something that means "password" or a placeholder name - either a variable or a placeholder
         # a literal value or config switch
-        - _?\)?(?:[a-zA-Z0-9._]+[_.])?(?:the )?(?:pass?(wo?r?d|code|phrase)|pass|pwd|secret|token|tok|redacted|placeholder|dummy|pw|thephrase)|write|read|on|off|true|false|none|null( \? )?|nil|undefined|eof|ignore|eol|git|yes|no|y|n),?\s*\){0,2}[\]>)]?(?:\)\s*\{)?\\?(( or | \|\| ).*)?$
+        - _?\)?(?:[a-zA-Z0-9._]+[_.])?(?:the )?(?:pass?(wo?r?d|code|phrase)|pass|pwd|secret|token|tok|redacted|placeholder|dummy|pw|thephrase|write|read|on|off|true|false|none|null( \? )?|nil|undefined|eof|ignore|eol|git|yes|no|y|n),?\s*\){0,2}[\]>)]?(?:\)\s*\{)?\\?(( or | \|\| ).*)?$
         # Python type hints, Swift typing
         - ^\s*(?:(?:typing\.)?(?:(?:[Tt]uple|[Ll]ist|[Dd]ict|Callable|Iterable|Sequence|Optional|Union)\[.*|(?:int|str|float|(?:typing.)?Any|None|bytes|bool|ReadableBuffer)\s*(?:[,|].*)?|(?:Int|Swift\.Int|Int32)\.*))\s*$
         # ..., \, , \n, \0, ',' and other single chars, smilies, hex, digits, nothing at all,

From 7bb93ce50432278667c35efc2e9149f7eb82d756 Mon Sep 17 00:00:00 2001
From: aegilops <41705651+aegilops@users.noreply.github.com>
Date: Wed, 10 Jul 2024 14:28:41 +0100
Subject: [PATCH 70/95] Updated README.md

---
 generic/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/generic/README.md b/generic/README.md
index e1916917..35a00819 100644
--- a/generic/README.md
+++ b/generic/README.md
@@ -59,7 +59,7 @@ Add these additional matches to the [Secret Scanning Custom Pattern](https://doc
 - Not Match:
 
   ```regex
-  _?\)?(?:[a-zA-Z0-9._]+[_.])?(?:the )?(?:pass?(wo?r?d|code|phrase)|pass|pwd|secret|token|tok|redacted|placeholder|dummy|pw|thephrase)|write|read|on|off|true|false|none|null( \? )?|nil|undefined|eof|ignore|eol|git|yes|no|y|n),?\s*\){0,2}[\]>)]?(?:\)\s*\{)?\\?(( or | \|\| ).*)?$
+  _?\)?(?:[a-zA-Z0-9._]+[_.])?(?:the )?(?:pass?(wo?r?d|code|phrase)|pass|pwd|secret|token|tok|redacted|placeholder|dummy|pw|thephrase|write|read|on|off|true|false|none|null( \? )?|nil|undefined|eof|ignore|eol|git|yes|no|y|n),?\s*\){0,2}[\]>)]?(?:\)\s*\{)?\\?(( or | \|\| ).*)?$
   ```
 - Not Match:
 

From a334644c622566eb35250d7734f9f7f29685a4d6 Mon Sep 17 00:00:00 2001
From: aegilops <41705651+aegilops@users.noreply.github.com>
Date: Wed, 10 Jul 2024 16:44:09 +0100
Subject: [PATCH 71/95] Make Okta precise pattern "smaller"

---
 vendors/patterns.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/vendors/patterns.yml b/vendors/patterns.yml
index 5c16a953..5002fc28 100644
--- a/vendors/patterns.yml
+++ b/vendors/patterns.yml
@@ -237,7 +237,7 @@ patterns:
       pattern: |
         0{2}[0-9A-Za-z_-]{40}
       start: |
-        (\bSSWS\s{1,5}|(?i)okta[_-]?(api[_-]?)?(token|key|secret)\s{0,32}([:=]|[=-]>|to|[!=]={1,2}|<>)\s{0,32}['"`]?)
+        (\bSSWS\s{1,5}|(?i)okta[_-]?(api[_-]?)?(token|key|secret)\s{0,28}([:=]|[=-]>|to|[!=]={1,2}|<>)\s{0,28}['"`]?)
       end: |
         \z|[^0-9A-Za-z_+/=-]
     test:

From 0e5a9d781d47df0fea40cd66778491bf20dc6b5d Mon Sep 17 00:00:00 2001
From: aegilops <41705651+aegilops@users.noreply.github.com>
Date: Wed, 10 Jul 2024 17:13:50 +0100
Subject: [PATCH 72/95] Fixed some offsets in test data

---
 configs/patterns.yml | 11 ++++++-----
 vendors/patterns.yml |  8 +++++---
 2 files changed, 11 insertions(+), 8 deletions(-)

diff --git a/configs/patterns.yml b/configs/patterns.yml
index b574b240..807462ae 100644
--- a/configs/patterns.yml
+++ b/configs/patterns.yml
@@ -173,8 +173,8 @@ patterns:
       - "Only allows for only single-quoted passwords, to better control false positives"
     test:
       data: "password: 'password123'"
-      start_offset: 10
-      end_offset: 21
+      start_offset: 11
+      end_offset: 22
 
 
   - name: YAML hardcoded passwords (double quoted strings)
@@ -201,8 +201,8 @@ patterns:
       data: |
         password: "password123"
 
-      start_offset: 10
-      end_offset: 21
+      start_offset: 11
+      end_offset: 22
 
 
   - name: YAML hardcoded passwords (multiline strings)
@@ -228,7 +228,8 @@ patterns:
           password123
           andsomemore
         other_field: foo
-      # TODO: add start_offset and end_offset
+      start_offset: 14
+      end_offset: 56
 
 
   - name: GitHub Actions SHA Checker
diff --git a/vendors/patterns.yml b/vendors/patterns.yml
index 5002fc28..d01bc6c5 100644
--- a/vendors/patterns.yml
+++ b/vendors/patterns.yml
@@ -556,7 +556,7 @@ patterns:
       data: |
         google_api_key: 0123456789abcdef0123456789abcdef01234567
       start_offset: 16
-      end_offset: -1
+      end_offset: 56
 
   - name: OpenStack password/API key
     type: openstack_password_or_key
@@ -610,7 +610,7 @@ patterns:
       data: |
         key: service:my-apollo-service:aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa
       start_offset: 5
-      end_offset: 71
+      end_offset: 67
 
   - name: ClickUp API key
     type: clickup_api_key
@@ -658,7 +658,7 @@ patterns:
       data: |
         jenkins_api_token="0123456789abcdef0123456789abcdef"
       start_offset: 19
-      end_offset: 52
+      end_offset: 51
 
   - name: AWS S3 presigned URL
     type: aws_s3_presigned_url
@@ -673,6 +673,8 @@ patterns:
     test:
       data: |
         https://snort-org-site.s3.amazonaws.com/production/document_files/files/000/000/596/original/Rules_Writers_Guide_to_Snort_3_Rules.pdf?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAIXACIED2SPMSC7GA%2F20211011%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20211011T165700Z&X-Amz-Expires=172800&X-Amz-SignedHeaders=host&X-Amz-Signature=1d5134d3a84db0f7b1ece95fcf98387a0cc32ff131b6c95e6a80d3cd4b616a5c
+      start_offset: 0
+      end_offset: 402
 
   - name: Azure Access Key (legacy format)
     type: azure_access_key_legacy

From 28de33c7c15bfe21df5b112133354ad20737f87c Mon Sep 17 00:00:00 2001
From: aegilops <41705651+aegilops@users.noreply.github.com>
Date: Wed, 10 Jul 2024 17:14:03 +0100
Subject: [PATCH 73/95] Updated README.md

---
 vendors/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/vendors/README.md b/vendors/README.md
index 50e0da16..17afb316 100644
--- a/vendors/README.md
+++ b/vendors/README.md
@@ -414,7 +414,7 @@ _version: v0.1_
 <summary>Start Pattern</summary>
 
 ```regex
-(\bSSWS\s{1,5}|(?i)okta[_-]?(api[_-]?)?(token|key|secret)\s{0,32}([:=]|[=-]>|to|[!=]={1,2}|<>)\s{0,32}['"`]?)
+(\bSSWS\s{1,5}|(?i)okta[_-]?(api[_-]?)?(token|key|secret)\s{0,28}([:=]|[=-]>|to|[!=]={1,2}|<>)\s{0,28}['"`]?)
 ```
 
 </details><details>

From 670078353b4fa76cbd288ed7f5957d5df25bd1f5 Mon Sep 17 00:00:00 2001
From: aegilops <41705651+aegilops@users.noreply.github.com>
Date: Wed, 10 Jul 2024 17:15:06 +0100
Subject: [PATCH 74/95] Fixed NO ID num offsets

---
 pii/patterns.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/pii/patterns.yml b/pii/patterns.yml
index 7b8f7ad3..f0924045 100644
--- a/pii/patterns.yml
+++ b/pii/patterns.yml
@@ -251,6 +251,8 @@ patterns:
     test:
       data: |
         41018512346
+      start_offset: 0
+      end_offset: 11
     comments:
       - With no validation of the checksum this can cause a lot of false positives
       - The example test data does not have a valid checksum - it is one of the examples used with one digit in the checksum changed

From 1ad022f02d0ecb8256aa461f9c9ecf1edc54f7b8 Mon Sep 17 00:00:00 2001
From: aegilops <41705651+aegilops@users.noreply.github.com>
Date: Wed, 10 Jul 2024 17:21:36 +0100
Subject: [PATCH 75/95] Remove false matches on just whitespace "passwords"

---
 vendors/patterns.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/vendors/patterns.yml b/vendors/patterns.yml
index d01bc6c5..b31ce592 100644
--- a/vendors/patterns.yml
+++ b/vendors/patterns.yml
@@ -563,7 +563,7 @@ patterns:
     description: OpenStack password or API key
     regex:
       pattern: |
-        [^'",\r\n\x00-\x08]+
+        [^'",\r\n \t\x00-\x08]+
       start: |
         (?i)OPEN_?STACK_(PASSWORD|API_?KEY)[_A-Z]*['"`]?(\s*[\]\)])?\s*([:,=]|[=-]>|to|[!=]={1,2}|<>)?\s*([[{])?['"`]?
       end: |
@@ -572,7 +572,7 @@ patterns:
         - ^(ENV|[a-z_]+)\[$
         - ^<%=.*%>$
         - ^([a-z_]+\.api_?key|self\.[a-z_]+|os\.environ\.get\()$
-        - ^(\$\{?[A-Z]+\}?|<password>)$
+        - ^(\$\{?[A-Z]+\}?|<password>|\s+)$
         - ^(@?[a-z_]+\[:.*\]|@[a-z_]+)$
     test:
       data: |

From 1030b7e1c31365335f21018cc24c95a7f74573c3 Mon Sep 17 00:00:00 2001
From: aegilops <41705651+aegilops@users.noreply.github.com>
Date: Wed, 10 Jul 2024 17:21:50 +0100
Subject: [PATCH 76/95] Updated README.md

---
 vendors/README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/vendors/README.md b/vendors/README.md
index 17afb316..c9432dab 100644
--- a/vendors/README.md
+++ b/vendors/README.md
@@ -1133,7 +1133,7 @@ _version: v0.1_
 <summary>Pattern Format</summary>
 
 ```regex
-[^'",\r\n\x00-\x08]+
+[^'",\r\n \t\x00-\x08]+
 ```
 
 </details>
@@ -1178,7 +1178,7 @@ Add these additional matches to the [Secret Scanning Custom Pattern](https://doc
 - Not Match:
 
   ```regex
-  ^(\$\{?[A-Z]+\}?|<password>)$
+  ^(\$\{?[A-Z]+\}?|<password>|\s+)$
   ```
 - Not Match:
 

From 8a16c38cf2cbf27e4c7d83668535a1eee66b5664 Mon Sep 17 00:00:00 2001
From: aegilops <41705651+aegilops@users.noreply.github.com>
Date: Wed, 10 Jul 2024 17:29:46 +0100
Subject: [PATCH 77/95] Fixed additional matches

---
 vendors/patterns.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/vendors/patterns.yml b/vendors/patterns.yml
index b31ce592..372aac4f 100644
--- a/vendors/patterns.yml
+++ b/vendors/patterns.yml
@@ -709,9 +709,9 @@ patterns:
       end: |
         \z|\s|['"`]
       additional_match:
-        - '[?&]sv=[0-9]{4}-[0-9]{2}-[0-9]{2}'
-        - '[?&]se=[0-9]{4}-[0-9]{2}-[0-9]{2}'
-        - '[?&]st=[0-9]{4}-[0-9]{2}-[0-9]{2}'
+        - '(^|&)sv=[0-9]{4}-[0-9]{2}-[0-9]{2}'
+        - '(^|&)se=[0-9]{4}-[0-9]{2}-[0-9]{2}'
+        - '(^|&)st=[0-9]{4}-[0-9]{2}-[0-9]{2}'
       additional_not_match:
         - '^https://files\.oaiusercontent\.com/'
     test:

From 2551dde4ff89e6c5b780459ac62603b837ed75a7 Mon Sep 17 00:00:00 2001
From: aegilops <41705651+aegilops@users.noreply.github.com>
Date: Wed, 10 Jul 2024 17:29:53 +0100
Subject: [PATCH 78/95] Updated README.md

---
 vendors/README.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/vendors/README.md b/vendors/README.md
index c9432dab..eed07b4c 100644
--- a/vendors/README.md
+++ b/vendors/README.md
@@ -1497,19 +1497,19 @@ Add these additional matches to the [Secret Scanning Custom Pattern](https://doc
 - Match:
 
   ```regex
-  [?&]sv=[0-9]{4}-[0-9]{2}-[0-9]{2}
+  (^|&)sv=[0-9]{4}-[0-9]{2}-[0-9]{2}
   ```
 
 - Match:
 
   ```regex
-  [?&]se=[0-9]{4}-[0-9]{2}-[0-9]{2}
+  (^|&)se=[0-9]{4}-[0-9]{2}-[0-9]{2}
   ```
 
 - Match:
 
   ```regex
-  [?&]st=[0-9]{4}-[0-9]{2}-[0-9]{2}
+  (^|&)st=[0-9]{4}-[0-9]{2}-[0-9]{2}
   ```
 - Not Match:
 

From a9e7fb81f0518c872448298ace33299c056203bc Mon Sep 17 00:00:00 2001
From: aegilops <41705651+aegilops@users.noreply.github.com>
Date: Wed, 10 Jul 2024 17:30:44 +0100
Subject: [PATCH 79/95] Fixed offset in test data

---
 vendors/patterns.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/vendors/patterns.yml b/vendors/patterns.yml
index 372aac4f..a6bc114a 100644
--- a/vendors/patterns.yml
+++ b/vendors/patterns.yml
@@ -717,8 +717,8 @@ patterns:
     test:
       data: |
         AZURE_SAS_TOKEN: "?sv=2019-12-12&ss=bfqt&srt=sco&sp=rwdlacupx&se=2021-10-11T16:57:00Z&st=2021-10-11T08:57:00Z&spr=https&sig=abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789abcdefghijklmn%3D"
-      start_offset: 18
-      end_offset: 169
+      start_offset: 19
+      end_offset: 214
     comments:
       - This is a Shared Access Signature (SAS) token for Azure services. See [these examples](https://learn.microsoft.com/en-us/rest/api/storageservices/service-sas-examples)
       - The token is a URL query string parameter, and the signature is a base64 encoded HMAC-SHA256 hash, so is a fixed length in plain text and always ends in =

From 96e992e1356c1244e4f73f92aae4b9cbfb09c9a4 Mon Sep 17 00:00:00 2001
From: aegilops <41705651+aegilops@users.noreply.github.com>
Date: Wed, 10 Jul 2024 17:32:09 +0100
Subject: [PATCH 80/95] Added missing escape of a bracket

---
 configs/patterns.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/configs/patterns.yml b/configs/patterns.yml
index 807462ae..e88ec24a 100644
--- a/configs/patterns.yml
+++ b/configs/patterns.yml
@@ -320,7 +320,7 @@ patterns:
         [\r\n#]|['"]\s*[\r\n]|\z
       additional_not_match:
         # variable, process substitution
-        - ^\$(\{[^}]+\}|\([^)]+)|[A-Za-z_]+|[0-9])$
+        - ^\$(\{[^}]+\}|\([^)]+\)|[A-Za-z_]+|[0-9])$
         # placeholders
         - ^(<[^>]+>|\[[^]+\]|\{[^}+\}|(your|my|the|a)_[a-z_]+|.*(passwor[t]|key|secret|token|密码).*|\.\.\.|xxx+|yyy+|zzz+|aaa+|bbb+|ccc+)$
         # non-secret related "key" content

From 5fe656a170cac38a99a3f0ba053b354a0f598bf7 Mon Sep 17 00:00:00 2001
From: aegilops <41705651+aegilops@users.noreply.github.com>
Date: Wed, 10 Jul 2024 17:32:16 +0100
Subject: [PATCH 81/95] Updated README.md

---
 configs/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/configs/README.md b/configs/README.md
index 1d6db976..90a73327 100644
--- a/configs/README.md
+++ b/configs/README.md
@@ -617,7 +617,7 @@ Add these additional matches to the [Secret Scanning Custom Pattern](https://doc
 - Not Match:
 
   ```regex
-  ^\$(\{[^}]+\}|\([^)]+)|[A-Za-z_]+|[0-9])$
+  ^\$(\{[^}]+\}|\([^)]+\)|[A-Za-z_]+|[0-9])$
   ```
 - Not Match:
 

From 5ccec15cecce4ba6e8fe282fad56c8bef06d2d3a Mon Sep 17 00:00:00 2001
From: aegilops <41705651+aegilops@users.noreply.github.com>
Date: Wed, 10 Jul 2024 17:34:53 +0100
Subject: [PATCH 82/95] Removed expectation to hit on obvious placeholder

---
 generic/patterns.yml | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/generic/patterns.yml b/generic/patterns.yml
index dc1143d6..ecc44228 100644
--- a/generic/patterns.yml
+++ b/generic/patterns.yml
@@ -37,9 +37,6 @@ patterns:
       - name: passwords.js
         start_offset: 41
         end_offset: 62
-      - name: passwords.js
-        start_offset: 97
-        end_offset: 112
     comments:
       - "Likely to cause large numbers of false positives - use with caution"
       - "`password`, `secret`, `key`, or password like prefix (fuzzy)"

From cd3cd9c968e02cb1e36fc89a88dcd015ac1a800f Mon Sep 17 00:00:00 2001
From: aegilops <41705651+aegilops@users.noreply.github.com>
Date: Wed, 10 Jul 2024 17:43:44 +0100
Subject: [PATCH 83/95] Fix missing CR/NL option in multiline YAML pattern

---
 configs/patterns.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/configs/patterns.yml b/configs/patterns.yml
index e88ec24a..2f8f8437 100644
--- a/configs/patterns.yml
+++ b/configs/patterns.yml
@@ -215,7 +215,7 @@ patterns:
       pattern: |
         [^\x00-\x08]+?
       start: |
-        (?:\n|\A)[ \t]*(?i)[a-z_-]*(?:secret|service_pass(wd|word|code|phrase)|pass(?:wd|word|code|phrase)?|key|token)[ \t]*:[ \t]*[|>][+-]?[ \t]*\n[ \t]+
+        (?:\n|\A)[ \t]*(?i)[a-z_-]*(?:secret|service_pass(wd|word|code|phrase)|pass(?:wd|word|code|phrase)?|key|token)[ \t]*:[ \t]*[|>][+-]?[ \t]*(\n|\r\n)[ \t]+
       end: |
         \n\n|\r\n\r\n|(\n|\r\n)[ \t]+\S+:|(\n|\r\n)\.\.\.[ \t\n\r]|\z
     comments:

From 2223b671bae5609762ff73a5957a3616256b618c Mon Sep 17 00:00:00 2001
From: aegilops <41705651+aegilops@users.noreply.github.com>
Date: Wed, 10 Jul 2024 17:43:53 +0100
Subject: [PATCH 84/95] Updated README.md

---
 configs/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/configs/README.md b/configs/README.md
index 90a73327..0dc306fe 100644
--- a/configs/README.md
+++ b/configs/README.md
@@ -419,7 +419,7 @@ _version: v0.1_
 <summary>Start Pattern</summary>
 
 ```regex
-(?:\n|\A)[ \t]*(?i)[a-z_-]*(?:secret|service_pass(wd|word|code|phrase)|pass(?:wd|word|code|phrase)?|key|token)[ \t]*:[ \t]*[|>][+-]?[ \t]*\n[ \t]+
+(?:\n|\A)[ \t]*(?i)[a-z_-]*(?:secret|service_pass(wd|word|code|phrase)|pass(?:wd|word|code|phrase)?|key|token)[ \t]*:[ \t]*[|>][+-]?[ \t]*(\n|\r\n)[ \t]+
 ```
 
 </details><details>

From af2968c4b4b6f50a43543fca795bd28283fb7db8 Mon Sep 17 00:00:00 2001
From: aegilops <41705651+aegilops@users.noreply.github.com>
Date: Wed, 10 Jul 2024 17:46:36 +0100
Subject: [PATCH 85/95] Try to fix shifting line endings issue

---
 configs/patterns.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/configs/patterns.yml b/configs/patterns.yml
index 2f8f8437..23dc46be 100644
--- a/configs/patterns.yml
+++ b/configs/patterns.yml
@@ -223,7 +223,7 @@ patterns:
       - "Some false positives in code or YAML files might appear, especially where the variable is called 'key' or 'token'"
       - "The pattern checks for certain key words to end the variable name (`secret`, `password`, etc.)"
     test:
-      data: |
+      data: |-
         password: |
           password123
           andsomemore

From cd1b892c7f2c5dcb6243ce1cd164aebfff5c2b79 Mon Sep 17 00:00:00 2001
From: aegilops <41705651+aegilops@users.noreply.github.com>
Date: Wed, 10 Jul 2024 18:00:24 +0100
Subject: [PATCH 86/95] Remove old Credit Card names from snapshots

---
 pii/__snapshots__/credit_card_discovery.csv  | 3 ---
 pii/__snapshots__/credit_card_mastercard.csv | 6 ------
 pii/__snapshots__/credit_card_visa.csv       | 4 ----
 3 files changed, 13 deletions(-)
 delete mode 100644 pii/__snapshots__/credit_card_discovery.csv
 delete mode 100644 pii/__snapshots__/credit_card_mastercard.csv
 delete mode 100644 pii/__snapshots__/credit_card_visa.csv

diff --git a/pii/__snapshots__/credit_card_discovery.csv b/pii/__snapshots__/credit_card_discovery.csv
deleted file mode 100644
index 3f703dee..00000000
--- a/pii/__snapshots__/credit_card_discovery.csv
+++ /dev/null
@@ -1,3 +0,0 @@
-secret_type,secret_type_display_name,secret,path,start_line,end_line,start_column,end_column
-"credit_card_discovery","Credit Card (Discovery)","19ff47cc8024c133d5845d3f8938caca289929031e7d508c3adf7adff177f0c2","pii/credit-cards.txt","46","46","1","17",
-"credit_card_discovery","Credit Card (Discovery)","d8086d483c15c711ebba19f966b97d3c2adcba74025ff8d7e07c3698c9531deb","pii/credit-cards.txt","47","47","1","17",
diff --git a/pii/__snapshots__/credit_card_mastercard.csv b/pii/__snapshots__/credit_card_mastercard.csv
deleted file mode 100644
index 18501dd1..00000000
--- a/pii/__snapshots__/credit_card_mastercard.csv
+++ /dev/null
@@ -1,6 +0,0 @@
-secret_type,secret_type_display_name,secret,path,start_line,end_line,start_column,end_column
-"credit_card_mastercard","Credit Card (MasterCard)","3aebd9be84d81a1c5676e31adda86653aa5e7da0a6bda5ea02fec6526c19bb11","pii/credit-cards.txt","14","14","1","17",
-"credit_card_mastercard","Credit Card (MasterCard)","334b48feec00e7ec9808e50f1f05efd5eb1089abce7430e3a5b2b79dd2da1b73","pii/credit-cards.txt","13","13","1","17",
-"credit_card_mastercard","Credit Card (MasterCard)","be6a3050f2611250d0b50c1fa6aff7d23925922a35520c2d0a51a262ba894f8e","pii/credit-cards.txt","12","12","1","17",
-"credit_card_mastercard","Credit Card (MasterCard)","304945e91de3deff52a61d08733141d72dd42ec9d47972f1060534d54c0c7f90","pii/credit-cards.txt","11","11","1","17",
-"credit_card_mastercard","Credit Card (MasterCard)","2f725bbd1f405a1ed0336abaf85ddfeb6902a9984a76fd877c3b5cc3b5085a82","pii/credit-cards.txt","10","10","1","17",
diff --git a/pii/__snapshots__/credit_card_visa.csv b/pii/__snapshots__/credit_card_visa.csv
deleted file mode 100644
index 8b2d9414..00000000
--- a/pii/__snapshots__/credit_card_visa.csv
+++ /dev/null
@@ -1,4 +0,0 @@
-secret_type,secret_type_display_name,secret,path,start_line,end_line,start_column,end_column
-"credit_card_visa","Credit Card (Visa)","253da55d8a05e361b2e41c28728129909ed9a235632f2c9100ea5e827e396db7","pii/credit-cards.txt","21","21","1","14",
-"credit_card_visa","Credit Card (Visa)","dd13cdf9af9dd3baf46ce96aecd7163cabf381ccb21e63f15f0fa10b1c663fa9","pii/credit-cards.txt","20","20","1","17",
-"credit_card_visa","Credit Card (Visa)","9bbef19476623ca56c17da75fd57734dbf82530686043a6e491c6d71befe8f6e","pii/credit-cards.txt","19","19","1","17",

From d8c0bb731cbb43eb78777f6df971e87a1cce0d67 Mon Sep 17 00:00:00 2001
From: aegilops <41705651+aegilops@users.noreply.github.com>
Date: Wed, 10 Jul 2024 17:18:40 +0000
Subject: [PATCH 87/95] Updated snapshots

---
 configs/__snapshots__/env_passwords.csv       |  1 +
 configs/__snapshots__/json_base64.csv         |  1 +
 configs/__snapshots__/json_base64_uri.csv     |  1 +
 configs/__snapshots__/json_token.csv          |  1 +
 configs/__snapshots__/yaml_base64.csv         |  1 +
 configs/__snapshots__/yaml_base64_uri.csv     |  1 +
 .../yaml_passwords_multiline.csv              |  1 +
 .../__snapshots__/yaml_passwords_plain.csv    |  1 +
 .../yaml_passwords_single_quoted.csv          |  1 +
 configs/__snapshots__/yaml_token.csv          |  1 +
 .../database_connection_string_full.csv       |  1 +
 .../__snapshots__/jdbc_connection_string.csv  |  1 +
 .../mongodb_connection_string.csv             |  1 +
 .../sqlalchemy_connection_string.csv          |  1 +
 generic/__snapshots__/generic_passwords.csv   | 56 +++++++++++++++++++
 .../generic_passwords_base64.csv              |  1 +
 .../generic_passwords_base64_uri.csv          |  1 +
 .../generic_passwords_fewer_fps.csv           |  1 +
 .../__snapshots__/generic_passwords_hex.csv   |  1 +
 pii/__snapshots__/credit_card_discovery.csv   |  2 -
 pii/__snapshots__/credit_card_mastercard.csv  |  5 --
 pii/__snapshots__/credit_card_visa.csv        |  3 -
 pii/__snapshots__/no_national_id_number.csv   |  1 +
 .../__snapshots__/alienvault_otx_api_key.csv  |  1 +
 .../__snapshots__/amazon_mws_auth_token.csv   |  1 +
 vendors/__snapshots__/apollo_io_api_key.csv   |  1 +
 .../__snapshots__/aws_s3_presigned_url.csv    |  1 +
 .../__snapshots__/azure_access_key_legacy.csv |  1 +
 vendors/__snapshots__/azure_sas_token.csv     |  1 +
 vendors/__snapshots__/circleci_api_token.csv  |  1 +
 vendors/__snapshots__/clickup_api_key.csv     |  1 +
 ...google_private_key_id_or_old_api_token.csv |  1 +
 vendors/__snapshots__/jenkins_api_token.csv   |  1 +
 vendors/__snapshots__/okta_token_precise.csv  |  1 +
 .../openstack_password_or_key.csv             |  1 +
 35 files changed, 87 insertions(+), 10 deletions(-)
 create mode 100644 configs/__snapshots__/env_passwords.csv
 create mode 100644 configs/__snapshots__/json_base64.csv
 create mode 100644 configs/__snapshots__/json_base64_uri.csv
 create mode 100644 configs/__snapshots__/json_token.csv
 create mode 100644 configs/__snapshots__/yaml_base64.csv
 create mode 100644 configs/__snapshots__/yaml_base64_uri.csv
 create mode 100644 configs/__snapshots__/yaml_passwords_multiline.csv
 create mode 100644 configs/__snapshots__/yaml_passwords_plain.csv
 create mode 100644 configs/__snapshots__/yaml_passwords_single_quoted.csv
 create mode 100644 configs/__snapshots__/yaml_token.csv
 create mode 100644 database/__snapshots__/database_connection_string_full.csv
 create mode 100644 database/__snapshots__/jdbc_connection_string.csv
 create mode 100644 database/__snapshots__/mongodb_connection_string.csv
 create mode 100644 database/__snapshots__/sqlalchemy_connection_string.csv
 create mode 100644 generic/__snapshots__/generic_passwords_base64.csv
 create mode 100644 generic/__snapshots__/generic_passwords_base64_uri.csv
 create mode 100644 generic/__snapshots__/generic_passwords_fewer_fps.csv
 create mode 100644 generic/__snapshots__/generic_passwords_hex.csv
 create mode 100644 pii/__snapshots__/no_national_id_number.csv
 create mode 100644 vendors/__snapshots__/alienvault_otx_api_key.csv
 create mode 100644 vendors/__snapshots__/amazon_mws_auth_token.csv
 create mode 100644 vendors/__snapshots__/apollo_io_api_key.csv
 create mode 100644 vendors/__snapshots__/aws_s3_presigned_url.csv
 create mode 100644 vendors/__snapshots__/azure_access_key_legacy.csv
 create mode 100644 vendors/__snapshots__/azure_sas_token.csv
 create mode 100644 vendors/__snapshots__/circleci_api_token.csv
 create mode 100644 vendors/__snapshots__/clickup_api_key.csv
 create mode 100644 vendors/__snapshots__/google_private_key_id_or_old_api_token.csv
 create mode 100644 vendors/__snapshots__/jenkins_api_token.csv
 create mode 100644 vendors/__snapshots__/okta_token_precise.csv
 create mode 100644 vendors/__snapshots__/openstack_password_or_key.csv

diff --git a/configs/__snapshots__/env_passwords.csv b/configs/__snapshots__/env_passwords.csv
new file mode 100644
index 00000000..384a50bb
--- /dev/null
+++ b/configs/__snapshots__/env_passwords.csv
@@ -0,0 +1 @@
+secret_type,secret_type_display_name,secret,path,start_line,end_line,start_column,end_column
diff --git a/configs/__snapshots__/json_base64.csv b/configs/__snapshots__/json_base64.csv
new file mode 100644
index 00000000..384a50bb
--- /dev/null
+++ b/configs/__snapshots__/json_base64.csv
@@ -0,0 +1 @@
+secret_type,secret_type_display_name,secret,path,start_line,end_line,start_column,end_column
diff --git a/configs/__snapshots__/json_base64_uri.csv b/configs/__snapshots__/json_base64_uri.csv
new file mode 100644
index 00000000..384a50bb
--- /dev/null
+++ b/configs/__snapshots__/json_base64_uri.csv
@@ -0,0 +1 @@
+secret_type,secret_type_display_name,secret,path,start_line,end_line,start_column,end_column
diff --git a/configs/__snapshots__/json_token.csv b/configs/__snapshots__/json_token.csv
new file mode 100644
index 00000000..384a50bb
--- /dev/null
+++ b/configs/__snapshots__/json_token.csv
@@ -0,0 +1 @@
+secret_type,secret_type_display_name,secret,path,start_line,end_line,start_column,end_column
diff --git a/configs/__snapshots__/yaml_base64.csv b/configs/__snapshots__/yaml_base64.csv
new file mode 100644
index 00000000..384a50bb
--- /dev/null
+++ b/configs/__snapshots__/yaml_base64.csv
@@ -0,0 +1 @@
+secret_type,secret_type_display_name,secret,path,start_line,end_line,start_column,end_column
diff --git a/configs/__snapshots__/yaml_base64_uri.csv b/configs/__snapshots__/yaml_base64_uri.csv
new file mode 100644
index 00000000..384a50bb
--- /dev/null
+++ b/configs/__snapshots__/yaml_base64_uri.csv
@@ -0,0 +1 @@
+secret_type,secret_type_display_name,secret,path,start_line,end_line,start_column,end_column
diff --git a/configs/__snapshots__/yaml_passwords_multiline.csv b/configs/__snapshots__/yaml_passwords_multiline.csv
new file mode 100644
index 00000000..384a50bb
--- /dev/null
+++ b/configs/__snapshots__/yaml_passwords_multiline.csv
@@ -0,0 +1 @@
+secret_type,secret_type_display_name,secret,path,start_line,end_line,start_column,end_column
diff --git a/configs/__snapshots__/yaml_passwords_plain.csv b/configs/__snapshots__/yaml_passwords_plain.csv
new file mode 100644
index 00000000..384a50bb
--- /dev/null
+++ b/configs/__snapshots__/yaml_passwords_plain.csv
@@ -0,0 +1 @@
+secret_type,secret_type_display_name,secret,path,start_line,end_line,start_column,end_column
diff --git a/configs/__snapshots__/yaml_passwords_single_quoted.csv b/configs/__snapshots__/yaml_passwords_single_quoted.csv
new file mode 100644
index 00000000..384a50bb
--- /dev/null
+++ b/configs/__snapshots__/yaml_passwords_single_quoted.csv
@@ -0,0 +1 @@
+secret_type,secret_type_display_name,secret,path,start_line,end_line,start_column,end_column
diff --git a/configs/__snapshots__/yaml_token.csv b/configs/__snapshots__/yaml_token.csv
new file mode 100644
index 00000000..384a50bb
--- /dev/null
+++ b/configs/__snapshots__/yaml_token.csv
@@ -0,0 +1 @@
+secret_type,secret_type_display_name,secret,path,start_line,end_line,start_column,end_column
diff --git a/database/__snapshots__/database_connection_string_full.csv b/database/__snapshots__/database_connection_string_full.csv
new file mode 100644
index 00000000..384a50bb
--- /dev/null
+++ b/database/__snapshots__/database_connection_string_full.csv
@@ -0,0 +1 @@
+secret_type,secret_type_display_name,secret,path,start_line,end_line,start_column,end_column
diff --git a/database/__snapshots__/jdbc_connection_string.csv b/database/__snapshots__/jdbc_connection_string.csv
new file mode 100644
index 00000000..384a50bb
--- /dev/null
+++ b/database/__snapshots__/jdbc_connection_string.csv
@@ -0,0 +1 @@
+secret_type,secret_type_display_name,secret,path,start_line,end_line,start_column,end_column
diff --git a/database/__snapshots__/mongodb_connection_string.csv b/database/__snapshots__/mongodb_connection_string.csv
new file mode 100644
index 00000000..384a50bb
--- /dev/null
+++ b/database/__snapshots__/mongodb_connection_string.csv
@@ -0,0 +1 @@
+secret_type,secret_type_display_name,secret,path,start_line,end_line,start_column,end_column
diff --git a/database/__snapshots__/sqlalchemy_connection_string.csv b/database/__snapshots__/sqlalchemy_connection_string.csv
new file mode 100644
index 00000000..384a50bb
--- /dev/null
+++ b/database/__snapshots__/sqlalchemy_connection_string.csv
@@ -0,0 +1 @@
+secret_type,secret_type_display_name,secret,path,start_line,end_line,start_column,end_column
diff --git a/generic/__snapshots__/generic_passwords.csv b/generic/__snapshots__/generic_passwords.csv
index 384a50bb..b0aaa6f8 100644
--- a/generic/__snapshots__/generic_passwords.csv
+++ b/generic/__snapshots__/generic_passwords.csv
@@ -1 +1,57 @@
 secret_type,secret_type_display_name,secret,path,start_line,end_line,start_column,end_column
+"generic_passwords","Generic Passwords","7c7853e3659d1c01e65f3cb460ac07d079288bfa5bc21aae3d31fe01a0814278","configs/application.properties","31","31","28","34",
+"generic_passwords","Generic Passwords","dbfdae0d66bb259d57896c533fa8d1d2bb5f3e685c3d1a32fbddc0109ebb13f8","common/top-passwords-shortlist.txt","24","24","10","16",
+"generic_passwords","Generic Passwords","8799e334b94c1c08dbc46b3887d86c1fb12a0080634d8ee72084ca536cb2342c","common/top-passwords-shortlist.txt","19","19","10","18",
+"generic_passwords","Generic Passwords","d90886c29bdf153471809be45596c83863e30586f10b7f38f157b965f8904981","common/top-passwords-shortlist.txt","23","23","10","16",
+"generic_passwords","Generic Passwords","9e6bbe76d8fafd22ff9293a04934361d2fcd709081272a005ab83ca5e66d6d16","database/tsql_create_user.sql","1","1","36","46",
+"generic_passwords","Generic Passwords","067d2f107733c6928d1699ae06855f1895747e6c0709cbc9807ecfe6c0c1c2dd","common/top-passwords-shortlist.txt","31","31","10","16",
+"generic_passwords","Generic Passwords","49924b9a100a8aa9dab6014398ed23d567f59036f14de87175387fbc02dba7cc","database/docker_sqlserver.yml","14","14","122","133",
+"generic_passwords","Generic Passwords","3820fe31b7bd9e2090f4a1f49c66f36006f8eedfb6552af35fe627bc070f27ae","configs/application.properties","29","29","28","35",
+"generic_passwords","Generic Passwords","66483b17db44f612efcb123bd998ed1f091b778aeb072f89f4b49bad370a7809","common/top-passwords-shortlist.txt","27","27","10","17",
+"generic_passwords","Generic Passwords","8d67877bcf2b6cba2f619aae5a3c6eff8210b74a26fa0466147622fc65237e2b","configs/application.properties","43","43","28","35",
+"generic_passwords","Generic Passwords","2eca97ad9e30f2ed064b3e6ecb2cd2e0ab8a05b5bbd54842df31abaa7477e989","configs/mysql/run-mysql.sh","5","5","28","42",
+"generic_passwords","Generic Passwords","2eca97ad9e30f2ed064b3e6ecb2cd2e0ab8a05b5bbd54842df31abaa7477e989","docker/mysql/run-mysql.sh","5","5","28","42",
+"generic_passwords","Generic Passwords","ffd0f84644936fceee82fffb1bb6056922126e05be040c84007c37a26fe6d9f2","common/top-passwords-shortlist.txt","4","4","10","17",
+"generic_passwords","Generic Passwords","5f07dd6c436eefbd1ee0a62f2353dab5523c2a6a7e007dd4dc04ad3cb225348e","configs/application.properties","50","50","28","35",
+"generic_passwords","Generic Passwords","90ff950e50d23ac032cf5705fc9b783fcf9c9a3d324b0c5e845a79da54d42ead","configs/application.properties","12","12","28","45",
+"generic_passwords","Generic Passwords","4813494d137e1631bba301d5acab6e7bb7aa74ce1185d456565ef51d737677b2","configs/application.properties","10","10","30","34",
+"generic_passwords","Generic Passwords","81ea24e895ec1e9ec08a8763a773b4253e1bc682bea01cb2a4e99b81e866c97d","common/top-passwords-shortlist.txt","14","14","10","16",
+"generic_passwords","Generic Passwords","544e62cee8033709e389e5b2755343d0d0fa8c4850215cfb6331717e80d1aea3","jwt/owasp-juice-shop.ts","207","207","20","52",
+"generic_passwords","Generic Passwords","9584c5ca00f5e717366a1c441d851c3daf00e4dd190b85e1cb0cb0d36af688de","configs/application.properties","36","36","28","35",
+"generic_passwords","Generic Passwords","24795a7bb3efee18ee90ad0dfbf6cf7f444de648e49a39b2657ab36dd2a8cbc4","common/top-passwords-shortlist.txt","32","32","10","16",
+"generic_passwords","Generic Passwords","066b91577bc547e21aa329c74d74b0e53e29534d4cc0ad455abba050121a9557","common/top-passwords-shortlist.txt","22","22","10","17",
+"generic_passwords","Generic Passwords","6ab5c67640a9b1f875050b158cdbb1d081ee76ff8e703279062ce8652a372332","configs/postgres/docker-compose.yml","12","12","95","102",
+"generic_passwords","Generic Passwords","b34e5475b55d84c7de90e07142d9427477c757a46a8e8712f7bc2f92dd6c6473","jwt/owasp-juice-shop.ts","43","43","20","29",
+"generic_passwords","Generic Passwords","e4ad93ca07acb8d908a3aa41e920ea4f4ef4f26e7f86cf8291c5db289780a5ae","common/top-passwords-shortlist.txt","18","18","10","18",
+"generic_passwords","Generic Passwords","9f214b49fafc432e116884a5244442203267f200e72c6ad3ad4dbfd7b0f089c3","jwt/owasp-juice-shop.ts","146","146","22","37",
+"generic_passwords","Generic Passwords","1e744b6a3177a1165c1d67ccfe8989267e8364189d9124aa87a756aa219db83c","configs/example.yml","5","5","15","38",
+"generic_passwords","Generic Passwords","fc93cb07e1ad92898527100e58a1cf1d1e7f65e9a266a6f87f3c84feb541c7b3","jwt/example.txt","2","2","12","15",
+"generic_passwords","Generic Passwords","e864c10ad586803d73b4e7a684ec1c8e2c2c9cd0a8718d2924bb50afe77c8d4b","common/top-passwords-shortlist.txt","28","28","10","15",
+"generic_passwords","Generic Passwords","b4b215eb04965939aab0e90f828c2e00cb17f147926b3f3de28901e3e02013ef","common/top-passwords-shortlist.txt","21","21","10","16",
+"generic_passwords","Generic Passwords","bdc7f0fb11236645c4c4d7aba2afab3649d5df400779061f1d034803a811c622","common/top-passwords-shortlist.txt","16","16","10","20",
+"generic_passwords","Generic Passwords","50b15954a84bfb3594a8ef96c10dd2fe25c27341db7c443ac1220ecb00f3f16a","common/top-passwords-shortlist.txt","36","36","10","16",
+"generic_passwords","Generic Passwords","20fdf64da3cd2c78ec3c033d2ac628bacf701711fa99435ee37bef0304800dc5","common/top-passwords-shortlist.txt","15","15","10","17",
+"generic_passwords","Generic Passwords","e2587f6c678061df35dbff7fee253348085045f1a4d68549f0e4261330af080c","configs/mysql/docker-compose.yml","8","8","28","51",
+"generic_passwords","Generic Passwords","e2587f6c678061df35dbff7fee253348085045f1a4d68549f0e4261330af080c","docker/mysql/docker-compose.yml","8","8","28","51",
+"generic_passwords","Generic Passwords","592c75c07fbc94b2eb5301bf67264dbd659a0f9c519b100d1f61db03e24dce77","common/top-passwords-shortlist.txt","2","2","10","15",
+"generic_passwords","Generic Passwords","406c0a9e14e3d00a364c559acde4192c6059163f7ef33e0820207e369338fa7b","common/top-passwords-shortlist.txt","29","29","10","17",
+"generic_passwords","Generic Passwords","1c20528f953116dfa0b8755308b7ee5ab14e5a2efdef2f90e65ce33526757351","common/top-passwords-shortlist.txt","6","6","10","16",
+"generic_passwords","Generic Passwords","1c8bfe8f801d79745c4631d09fff36c82aa37fc4cce4fc946683d7b336b63032","common/top-passwords-shortlist.txt","13","13","10","17",
+"generic_passwords","Generic Passwords","09d73fcb2ef34f11781d8f35d88428b9b90c3431dc82417ad161655cf1c465ed","common/top-passwords-shortlist.txt","17","17","10","16",
+"generic_passwords","Generic Passwords","2b08a09b43e6d9e6a51a2ad5fcdfc6532e95da6d72cd0ef5f65d8e6ae86ca3e5","common/top-passwords-shortlist.txt","34","34","10","16",
+"generic_passwords","Generic Passwords","ddd9a2b20bba26222d4d886042639f855ddce06f41a8c5df391b79fd15367ccc","common/top-passwords-shortlist.txt","33","33","10","16",
+"generic_passwords","Generic Passwords","8bb0cf6eb9b17d0f7d22b456f121257dc1254e1f01665370476383ea776df414","common/top-passwords-shortlist.txt","7","7","10","17",
+"generic_passwords","Generic Passwords","cb1b7759c2c93af5a9359769ec4fa443112b4b7c145e99767c16620aaa3c2c6c","configs/mysql/docker-compose.yml","11","11","23","36",
+"generic_passwords","Generic Passwords","cb1b7759c2c93af5a9359769ec4fa443112b4b7c145e99767c16620aaa3c2c6c","docker/mysql/docker-compose.yml","11","11","23","36",
+"generic_passwords","Generic Passwords","181b99e27e562b1237d009179d010d5774a853606551d7ac983b8c3daa9c5163","generic/passwords.js","3","3","13","34",
+"generic_passwords","Generic Passwords","ae9216f925f35e1784e790ba36020358acd29bbc0a8087ce5caf72d31560372f","common/top-passwords-shortlist.txt","26","26","10","15",
+"generic_passwords","Generic Passwords","b45f549d9212dac4edc9a16b6d18246fb5dae1c9c0d1d830a75b54573c06ee12","common/top-passwords-shortlist.txt","12","12","10","16",
+"generic_passwords","Generic Passwords","bef57ec7f53a6d40beb640a780a639c83bc29ac8a9816f1fc6c5c6dcd93c4721","common/top-passwords-shortlist.txt","8","8","10","16",
+"generic_passwords","Generic Passwords","f081f7b8d4310e67a7572f60b6070a3034d5f1ae1465b3fe4f8dafca9213a0e3","common/top-passwords-shortlist.txt","30","30","10","18",
+"generic_passwords","Generic Passwords","04d116726bdc3b35faf64ccd34af64c5308b3433e35b33db06219f071275fde8","common/top-passwords-shortlist.txt","20","20","10","18",
+"generic_passwords","Generic Passwords","1fb8edf1e0156e7eb349146b4c1f91ea88821c269bb290afeb21b90ef1bb379c","generic/passwords.js","1","1","15","26",
+"generic_passwords","Generic Passwords","d451823f5c7ea01b991edb52a7890624607a24f1b8e00f2e89f6807433f079ec","configs/application.properties","14","14","29","49",
+"generic_passwords","Generic Passwords","65e84be33532fb784c48129675f9eff3a682b27168c0ea744b2cf58ee02337c5","common/top-passwords-shortlist.txt","11","11","10","16",
+"generic_passwords","Generic Passwords","0fb769916f2a1adbca016f75e580284fd7bc26ab6e91e094d1096681445f62d3","generic/passwords.js","7","7","16","31",
+"generic_passwords","Generic Passwords","464c7a646393b68d1a42076c010b5aae418d8d322f233ca0b8cd8e2c6bcd9676","common/top-passwords-shortlist.txt","10","10","10","14",
+"generic_passwords","Generic Passwords","c471039f93e753cbc8c09509e9f937af8633c7a595267536714db11916779c81","common/top-passwords-shortlist.txt","3","3","10","19",
diff --git a/generic/__snapshots__/generic_passwords_base64.csv b/generic/__snapshots__/generic_passwords_base64.csv
new file mode 100644
index 00000000..384a50bb
--- /dev/null
+++ b/generic/__snapshots__/generic_passwords_base64.csv
@@ -0,0 +1 @@
+secret_type,secret_type_display_name,secret,path,start_line,end_line,start_column,end_column
diff --git a/generic/__snapshots__/generic_passwords_base64_uri.csv b/generic/__snapshots__/generic_passwords_base64_uri.csv
new file mode 100644
index 00000000..384a50bb
--- /dev/null
+++ b/generic/__snapshots__/generic_passwords_base64_uri.csv
@@ -0,0 +1 @@
+secret_type,secret_type_display_name,secret,path,start_line,end_line,start_column,end_column
diff --git a/generic/__snapshots__/generic_passwords_fewer_fps.csv b/generic/__snapshots__/generic_passwords_fewer_fps.csv
new file mode 100644
index 00000000..384a50bb
--- /dev/null
+++ b/generic/__snapshots__/generic_passwords_fewer_fps.csv
@@ -0,0 +1 @@
+secret_type,secret_type_display_name,secret,path,start_line,end_line,start_column,end_column
diff --git a/generic/__snapshots__/generic_passwords_hex.csv b/generic/__snapshots__/generic_passwords_hex.csv
new file mode 100644
index 00000000..384a50bb
--- /dev/null
+++ b/generic/__snapshots__/generic_passwords_hex.csv
@@ -0,0 +1 @@
+secret_type,secret_type_display_name,secret,path,start_line,end_line,start_column,end_column
diff --git a/pii/__snapshots__/credit_card_discovery.csv b/pii/__snapshots__/credit_card_discovery.csv
index 3f703dee..384a50bb 100644
--- a/pii/__snapshots__/credit_card_discovery.csv
+++ b/pii/__snapshots__/credit_card_discovery.csv
@@ -1,3 +1 @@
 secret_type,secret_type_display_name,secret,path,start_line,end_line,start_column,end_column
-"credit_card_discovery","Credit Card (Discovery)","19ff47cc8024c133d5845d3f8938caca289929031e7d508c3adf7adff177f0c2","pii/credit-cards.txt","46","46","1","17",
-"credit_card_discovery","Credit Card (Discovery)","d8086d483c15c711ebba19f966b97d3c2adcba74025ff8d7e07c3698c9531deb","pii/credit-cards.txt","47","47","1","17",
diff --git a/pii/__snapshots__/credit_card_mastercard.csv b/pii/__snapshots__/credit_card_mastercard.csv
index 18501dd1..384a50bb 100644
--- a/pii/__snapshots__/credit_card_mastercard.csv
+++ b/pii/__snapshots__/credit_card_mastercard.csv
@@ -1,6 +1 @@
 secret_type,secret_type_display_name,secret,path,start_line,end_line,start_column,end_column
-"credit_card_mastercard","Credit Card (MasterCard)","3aebd9be84d81a1c5676e31adda86653aa5e7da0a6bda5ea02fec6526c19bb11","pii/credit-cards.txt","14","14","1","17",
-"credit_card_mastercard","Credit Card (MasterCard)","334b48feec00e7ec9808e50f1f05efd5eb1089abce7430e3a5b2b79dd2da1b73","pii/credit-cards.txt","13","13","1","17",
-"credit_card_mastercard","Credit Card (MasterCard)","be6a3050f2611250d0b50c1fa6aff7d23925922a35520c2d0a51a262ba894f8e","pii/credit-cards.txt","12","12","1","17",
-"credit_card_mastercard","Credit Card (MasterCard)","304945e91de3deff52a61d08733141d72dd42ec9d47972f1060534d54c0c7f90","pii/credit-cards.txt","11","11","1","17",
-"credit_card_mastercard","Credit Card (MasterCard)","2f725bbd1f405a1ed0336abaf85ddfeb6902a9984a76fd877c3b5cc3b5085a82","pii/credit-cards.txt","10","10","1","17",
diff --git a/pii/__snapshots__/credit_card_visa.csv b/pii/__snapshots__/credit_card_visa.csv
index 8b2d9414..384a50bb 100644
--- a/pii/__snapshots__/credit_card_visa.csv
+++ b/pii/__snapshots__/credit_card_visa.csv
@@ -1,4 +1 @@
 secret_type,secret_type_display_name,secret,path,start_line,end_line,start_column,end_column
-"credit_card_visa","Credit Card (Visa)","253da55d8a05e361b2e41c28728129909ed9a235632f2c9100ea5e827e396db7","pii/credit-cards.txt","21","21","1","14",
-"credit_card_visa","Credit Card (Visa)","dd13cdf9af9dd3baf46ce96aecd7163cabf381ccb21e63f15f0fa10b1c663fa9","pii/credit-cards.txt","20","20","1","17",
-"credit_card_visa","Credit Card (Visa)","9bbef19476623ca56c17da75fd57734dbf82530686043a6e491c6d71befe8f6e","pii/credit-cards.txt","19","19","1","17",
diff --git a/pii/__snapshots__/no_national_id_number.csv b/pii/__snapshots__/no_national_id_number.csv
new file mode 100644
index 00000000..384a50bb
--- /dev/null
+++ b/pii/__snapshots__/no_national_id_number.csv
@@ -0,0 +1 @@
+secret_type,secret_type_display_name,secret,path,start_line,end_line,start_column,end_column
diff --git a/vendors/__snapshots__/alienvault_otx_api_key.csv b/vendors/__snapshots__/alienvault_otx_api_key.csv
new file mode 100644
index 00000000..384a50bb
--- /dev/null
+++ b/vendors/__snapshots__/alienvault_otx_api_key.csv
@@ -0,0 +1 @@
+secret_type,secret_type_display_name,secret,path,start_line,end_line,start_column,end_column
diff --git a/vendors/__snapshots__/amazon_mws_auth_token.csv b/vendors/__snapshots__/amazon_mws_auth_token.csv
new file mode 100644
index 00000000..384a50bb
--- /dev/null
+++ b/vendors/__snapshots__/amazon_mws_auth_token.csv
@@ -0,0 +1 @@
+secret_type,secret_type_display_name,secret,path,start_line,end_line,start_column,end_column
diff --git a/vendors/__snapshots__/apollo_io_api_key.csv b/vendors/__snapshots__/apollo_io_api_key.csv
new file mode 100644
index 00000000..384a50bb
--- /dev/null
+++ b/vendors/__snapshots__/apollo_io_api_key.csv
@@ -0,0 +1 @@
+secret_type,secret_type_display_name,secret,path,start_line,end_line,start_column,end_column
diff --git a/vendors/__snapshots__/aws_s3_presigned_url.csv b/vendors/__snapshots__/aws_s3_presigned_url.csv
new file mode 100644
index 00000000..384a50bb
--- /dev/null
+++ b/vendors/__snapshots__/aws_s3_presigned_url.csv
@@ -0,0 +1 @@
+secret_type,secret_type_display_name,secret,path,start_line,end_line,start_column,end_column
diff --git a/vendors/__snapshots__/azure_access_key_legacy.csv b/vendors/__snapshots__/azure_access_key_legacy.csv
new file mode 100644
index 00000000..384a50bb
--- /dev/null
+++ b/vendors/__snapshots__/azure_access_key_legacy.csv
@@ -0,0 +1 @@
+secret_type,secret_type_display_name,secret,path,start_line,end_line,start_column,end_column
diff --git a/vendors/__snapshots__/azure_sas_token.csv b/vendors/__snapshots__/azure_sas_token.csv
new file mode 100644
index 00000000..384a50bb
--- /dev/null
+++ b/vendors/__snapshots__/azure_sas_token.csv
@@ -0,0 +1 @@
+secret_type,secret_type_display_name,secret,path,start_line,end_line,start_column,end_column
diff --git a/vendors/__snapshots__/circleci_api_token.csv b/vendors/__snapshots__/circleci_api_token.csv
new file mode 100644
index 00000000..384a50bb
--- /dev/null
+++ b/vendors/__snapshots__/circleci_api_token.csv
@@ -0,0 +1 @@
+secret_type,secret_type_display_name,secret,path,start_line,end_line,start_column,end_column
diff --git a/vendors/__snapshots__/clickup_api_key.csv b/vendors/__snapshots__/clickup_api_key.csv
new file mode 100644
index 00000000..384a50bb
--- /dev/null
+++ b/vendors/__snapshots__/clickup_api_key.csv
@@ -0,0 +1 @@
+secret_type,secret_type_display_name,secret,path,start_line,end_line,start_column,end_column
diff --git a/vendors/__snapshots__/google_private_key_id_or_old_api_token.csv b/vendors/__snapshots__/google_private_key_id_or_old_api_token.csv
new file mode 100644
index 00000000..384a50bb
--- /dev/null
+++ b/vendors/__snapshots__/google_private_key_id_or_old_api_token.csv
@@ -0,0 +1 @@
+secret_type,secret_type_display_name,secret,path,start_line,end_line,start_column,end_column
diff --git a/vendors/__snapshots__/jenkins_api_token.csv b/vendors/__snapshots__/jenkins_api_token.csv
new file mode 100644
index 00000000..384a50bb
--- /dev/null
+++ b/vendors/__snapshots__/jenkins_api_token.csv
@@ -0,0 +1 @@
+secret_type,secret_type_display_name,secret,path,start_line,end_line,start_column,end_column
diff --git a/vendors/__snapshots__/okta_token_precise.csv b/vendors/__snapshots__/okta_token_precise.csv
new file mode 100644
index 00000000..384a50bb
--- /dev/null
+++ b/vendors/__snapshots__/okta_token_precise.csv
@@ -0,0 +1 @@
+secret_type,secret_type_display_name,secret,path,start_line,end_line,start_column,end_column
diff --git a/vendors/__snapshots__/openstack_password_or_key.csv b/vendors/__snapshots__/openstack_password_or_key.csv
new file mode 100644
index 00000000..384a50bb
--- /dev/null
+++ b/vendors/__snapshots__/openstack_password_or_key.csv
@@ -0,0 +1 @@
+secret_type,secret_type_display_name,secret,path,start_line,end_line,start_column,end_column

From 65a196eb05ac3476c14da90a966488597d3d69d9 Mon Sep 17 00:00:00 2001
From: aegilops <41705651+aegilops@users.noreply.github.com>
Date: Thu, 11 Jul 2024 16:46:28 +0100
Subject: [PATCH 88/95] More attempts to remove FPs from generic passwords

---
 configs/patterns.yml | 23 ++++++++++++-----------
 generic/patterns.yml | 18 +++++++++---------
 2 files changed, 21 insertions(+), 20 deletions(-)

diff --git a/configs/patterns.yml b/configs/patterns.yml
index 23dc46be..79d0fdf1 100644
--- a/configs/patterns.yml
+++ b/configs/patterns.yml
@@ -97,11 +97,11 @@ patterns:
       end: |
         ['"\r\n]|\z
       additional_not_match:
-        - ^(?:keyPassphrase|password|key|[ \t]+|\$\{[^}]+}|(?:str|string|int|bool)( +#.*)?),?$
-        - ^(?:.* = )?(?:None|[Tt]rue|[Ff]alse|[Nn]ull|Default(?:Type)?|Event|[A-Z]+_KEY|VERSION|NAME|update|destroy|(?:dis|en)ableEventListeners|\.\.\.),?$
-        - ^(?:(?:this|self|obj)\.)(?:[A-Za-z_]+\,|[A-Za-z_].*)$
-        - ^(?:(?:[a-zA-Z_]+(?:\(\))?\.)*[a-zA-Z_]+\(\)|\|\s*)$|\{\{[^}]+\}\}|\$\{\{ |^!Ref 
-        - ^\s*(?:typing\.)?(?:[Tt]uple|[Ll]ist|[Dd]ict|Callable|Iterable|Sequence|Optional|Union)\[.*$
+        - ^(keyPassphrase|password|key|[ \t]+|\$\{[^}]+}|(str|string|int|bool)( +#.*)?),?$
+        - ^(.* = )?(None|[Tt]rue|[Ff]alse|[Nn]ull|Default(Type)?|Event|[A-Z]+_KEY|VERSION|NAME|update|destroy|(?:dis|en)ableEventListeners|\.\.\.),?$
+        - ^(((this|self|obj)\.)([A-Za-z_]+\,|[A-Za-z_].*)|\{\}|\[\]|[0-9a-zA-Z],|\{)$
+        - ^(([a-zA-Z_]+(\(\))?\.)*[a-zA-Z_]+\(\)|\|\s*)$|\{\{[^}]+\}\}|\$\{\{ |^!Ref |^\$\(.*\)|^(https?|file)://|^/[a-zA-Z0-9.-/]+/[a-zA-Z0-9]{5,}\.(pem|crt|key)$|^=|\(\) => |\([^)]+\) => \{
+        - ^\s*(typing\.)?([Tt]uple|[Ll]ist|[Dd]ict|Callable|Iterable|Sequence|Optional|Union)\[.*$
     comments:
       - "Expect large numbers of false positives on variables containing 'key' or 'token'"
       - "The hardcoded password is any length"
@@ -135,11 +135,11 @@ patterns:
       end: |
         [\r\n]|\z
       additional_not_match:
-        - ^(?:keyPassphrase|password|key|[ \t]+|\$\{[^}]+}|(?:str|string|int|bool)( +#.*)?),?$
-        - ^(?:.* = )?(?:None|[Tt]rue|[Ff]alse|[Nn]ull|Default(?:Type)?|Event|[A-Z]+_KEY|VERSION|NAME|update|destroy|(?:dis|en)ableEventListeners|\.\.\.),?$
-        - '^(?:(?:(?:this|self|obj)\.)(?:[A-Za-z_]+\,|[A-Za-z_].*)|([!&*{}[\],#|>@`"''%]|[:?-] ).*)$'
-        - ^(?:[a-zA-Z_]+(?:\(\))?\.)*[a-zA-Z_]+\(\)$|\$\{\{[^}]+\}\}
-        - ^\s*(?:typing\.)?(?:[Tt]uple|[Ll]ist|[Dd]ict|Callable|Iterable|Sequence|Optional|Union)\[.*$
+        - ^(keyPassphrase|password|key|[ \t]+|\$\{[^}]+}|(str|string|int|bool)( +#.*)?),?$
+        - ^(.* = )?(None|[Tt]rue|[Ff]alse|[Nn]ull|Default(Type)?|Event|[A-Z]+_KEY|VERSION|NAME|update|destroy|(?:dis|en)ableEventListeners|\.\.\.),?$
+        - ^(((this|self|obj)\.)([A-Za-z_]+\,|[A-Za-z_].*)|\{\}|\[\]|[0-9a-zA-Z],|\{)$
+        - ^(([a-zA-Z_]+(\(\))?\.)*[a-zA-Z_]+\(\)|\|\s*)$|\{\{[^}]+\}\}|\$\{\{ |^!Ref |^\$\(.*\)|^(https?|file)://|^/[a-zA-Z0-9.-/]+/[a-zA-Z0-9]{5,}\.(pem|crt|key)$|^=|\(\) => |\([^)]+\) => \{
+        - ^\s*(typing\.)?([Tt]uple|[Ll]ist|[Dd]ict|Callable|Iterable|Sequence|Optional|Union)\[.*$
     comments:
       - "The hardcoded password is any length"
       - "Some false positives in code might appear"
@@ -313,7 +313,7 @@ patterns:
     regex:
       version: 0.1
       pattern: |
-        [^\r\n\x00-\x08'"#]+
+        [^\r\n\x00-\x08'"#]{8,}
       start: |
         (?:\n|\A)((export|ENV|ARG) )?[A-Z_]*(?:SECRET|PASS(?:WD|WOR[TD]|CODE|PHRASE)?|KEY|TOKEN)=['"]?
       end: |
@@ -335,6 +335,7 @@ patterns:
       - "Some false positives in code might appear, especially where the variable name is 'key' or 'token'"
       - "The pattern only checks for certain key words to begin the pattern (`secret`, `password`, etc.)"
       - "More restrictive than the Generic Passwords pattern, so less prone to false positives"
+      - "Lower length limit of 8 to remove FPs"
     test:
       data: |
         SECRET_KEY=password123
diff --git a/generic/patterns.yml b/generic/patterns.yml
index ecc44228..8f5707b5 100644
--- a/generic/patterns.yml
+++ b/generic/patterns.yml
@@ -15,17 +15,17 @@ patterns:
       additional_not_match:
         # something that means "password" or a placeholder name - either a variable or a placeholder
         # a literal value or config switch
-        - _?\)?(?:[a-zA-Z0-9._]+[_.])?(?:the )?(?:pass?(wo?r?d|code|phrase)|pass|pwd|secret|token|tok|redacted|placeholder|dummy|pw|thephrase|write|read|on|off|true|false|none|null( \? )?|nil|undefined|eof|ignore|eol|git|yes|no|y|n),?\s*\){0,2}[\]>)]?(?:\)\s*\{)?\\?(( or | \|\| ).*)?$
+        - (?i)_?\)?(?:[a-zA-Z0-9._]+[_.])?(?:the )?(?:pass?(wo?r?d|code|phrase)|pass|pwd|secret|token|key|tok|redacted|placeholder|dummy|pw|thephrase|write|read|on|off|true|false|none|value|null( \? )?|nil|undefined|eof|ignore|eol|git|yes|no|y|n|f[0-9]{1,2}|[a-zA-Z]),?\s*\){0,2}[\]>)]?(?:\)\s*\{)?\\?(( or | \|\| ).*)?$
         # Python type hints, Swift typing
         - ^\s*(?:(?:typing\.)?(?:(?:[Tt]uple|[Ll]ist|[Dd]ict|Callable|Iterable|Sequence|Optional|Union)\[.*|(?:int|str|float|(?:typing.)?Any|None|bytes|bool|ReadableBuffer)\s*(?:[,|].*)?|(?:Int|Swift\.Int|Int32)\.*))\s*$
         # ..., \, , \n, \0, ',' and other single chars, smilies, hex, digits, nothing at all,
         # directories, regex, format string placeholder, urllib demo passphrase, "optional" in docs, a variable substitution, or surrounded by brackets of various kinds
         # all with possible ',' and surrounding whitespace, possibly with a following comment
-        - ^\s*(?:\.\.\.|\\|\\n|\\0|\?|\$\(|[,()[\]{}`.]\\?|-[)(]|\\f21b|0x[A-Fa-f0-9]+|[0-9]{1,4}|(?:~|/tmp|\.\.|\.)|\\{1,2}w\+/g,( \\?)?|%[sr]|geheim\$parole|\([Oo]ptional\).*|\$?(?:\{\{?[^}]+\}\}?|\(\(?[^)]+\)\)?|\[\[?[^\]+]\]\]?)|(before|hover|focus)(,| \{))?,?\s*(?:\s*(?:/\*|#|//).*)?$
+        - ^\s*(?:\.\.\.|\\|\\n|\\0|\?|\$\(|[,()[\]{}`.]\\?|-[)(]|\\f21b|0x[A-Fa-f0-9]+|[0-9]{1,4}|(?:~|/tmp|\.\.|\.|/[a-zA-Z/.-]+/[a-zA-Z0-9]+\.(pem|crt|key))|\\{1,2}w\+/g,( \\?)?|%[sr]|geheim\$parole|\([Oo]ptional\).*|\$?(?:\{\{?[^}]+\}\}?|\(\(?[^)]+\)\)?|\[\[?[^\]+]\]\]?)|(before|hover|focus)(,| \{))?,?\s*(?:\s*(?:/\*|#|//).*)?$
         # function definitions, e.g. Javascript, function calls or variable declaration
-        - ^(?:function\s*\([^)]*\)\s*{\s*.*|\([^)]*\)\s*=>\s*(?:{\s*|[^;)]+[;)])|(?:new |\([A-Za-z]+\)\s*)?[a-zA-Z0-9_.]+\s*\(.*|(?:public|private) [A-Za-z0-9_]+ \{|[A-Za-z0-9_.-]+\s*\) \{)$|\{\{[^}]+\}\}|\$\{\{ 
+        - ^(?:function\s*\([^)]*\)\s*{\s*.*|\([^)]*\)\s*=>\s*(?:{\s*|[^;)]+[;)])|(?:new |\([A-Za-z]+\)\s*)?[a-zA-Z0-9_.]+\s*\(.*|(?:public|private) [A-Za-z0-9_]+ \{|[A-Za-z0-9_.-]+\s*\) \{)$|\{\{[^}]+\}\}|\$\{\{|\{\}$|\[\]$|(0x)?%[0-9]+x|%[dusx]\.$
         # reference to a member variable, index into a variable, bash variables, perl hash key index, environment vars
-        - ^\s*(?:(?:self|this)\.[a-zA-Z_][a-zA-Z0-9_.]+[,[]?|[a-zA-Z0-9_.]+\[(?:[a-zA-Z0-9_.]+)?\]?|\$(?:[1-9]|[A-Za-z0-9_]+)\{?|os\.environ\[[^\]]\]|process\.env\.[A-Z0-9_]+)\s*(?:,|\|\||&&)?\s*$
+        - ^\s*(?:(?:self|this)\.[a-zA-Z_][a-zA-Z0-9_.]+[,[]?|[a-zA-Z0-9_.]+\[(?:[a-zA-Z0-9_.]+)?\]?|\$(?:[1-9]|[A-Za-z0-9_]+)\{?|os\.environ\[[^\]]\]|process\.env\.[A-Z0-9_]+)\s*(?:,|\|\||&&)?\s*$|`(%s|\+)
     test:
       data: password=Password123
       start_offset: 9
@@ -182,7 +182,7 @@ patterns:
     regex:
       version: 0.2
       pattern: |
-        [a-zA-Z0-9_.=/+:-]+
+        [a-zA-Z0-9_.=/+:-]{12,}
       start: |
         (Authorization: |['"])([Bb]earer |[Tt]oken (token=)?)
       end: |
@@ -191,8 +191,7 @@ patterns:
         - ^(?:letmein|Oracle|SuperSecretString|foo|ababbdbbebbbebdbbe5538003023|XYZ_INVALID_ACCESTOKEN_XYZ|QQ==|Shizuku|mF_9.B5f-4.1JqM|h480djs93hd8|SlAV32hkKG|YmVlcDpib29w)$
         - ^(?i)(?:dummy|fake|bearer|auth|invalid|your|my|the|undefined|github|oidc|database)(?:_api)?(?:_?token|key|secret)?$
         - ^(?i)(?:[a-z0-9]|XYZ|ABC|123|.*_token)$
-        - (?i)x{5}
-        - ^(?i)(x+|y+|z+|a+|\.+|.*\.\.\.)$
+        - (^(?i)(x+|y+|z+|a+|\.+|.*\.\.\.)$|(?i)x{5}
     expected:
       - name: bearer.txt
         start_offset: 45
@@ -205,8 +204,9 @@ patterns:
         start_offset: 23
         end_offset: 42
     comments:
-      - "As used in an Authorization header"
-      - "We try to remove common placeholders"
+      - As used in an Authorization header
+      - We try to remove common placeholders
+      - Lower length limit of 12 to remove common false positives on "Token <common word>", since most words are below 12 characters in length
 
 
   - name: OAuth client secret and ID pair

From b9a4d5d5c4a5a6447f45c926132b390d58bb64d0 Mon Sep 17 00:00:00 2001
From: aegilops <41705651+aegilops@users.noreply.github.com>
Date: Thu, 11 Jul 2024 16:48:45 +0100
Subject: [PATCH 89/95] Missing brace

---
 generic/patterns.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/generic/patterns.yml b/generic/patterns.yml
index 8f5707b5..53e6c202 100644
--- a/generic/patterns.yml
+++ b/generic/patterns.yml
@@ -191,7 +191,7 @@ patterns:
         - ^(?:letmein|Oracle|SuperSecretString|foo|ababbdbbebbbebdbbe5538003023|XYZ_INVALID_ACCESTOKEN_XYZ|QQ==|Shizuku|mF_9.B5f-4.1JqM|h480djs93hd8|SlAV32hkKG|YmVlcDpib29w)$
         - ^(?i)(?:dummy|fake|bearer|auth|invalid|your|my|the|undefined|github|oidc|database)(?:_api)?(?:_?token|key|secret)?$
         - ^(?i)(?:[a-z0-9]|XYZ|ABC|123|.*_token)$
-        - (^(?i)(x+|y+|z+|a+|\.+|.*\.\.\.)$|(?i)x{5}
+        - (^(?i)(x+|y+|z+|a+|\.+|.*\.\.\.)$|(?i)x{5})
     expected:
       - name: bearer.txt
         start_offset: 45

From 37b26409ea8322b62ec4fdddee1f9afbc034e70e Mon Sep 17 00:00:00 2001
From: aegilops <41705651+aegilops@users.noreply.github.com>
Date: Thu, 11 Jul 2024 16:54:17 +0100
Subject: [PATCH 90/95] Changed NOT match for generic PW

---
 generic/patterns.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/generic/patterns.yml b/generic/patterns.yml
index 53e6c202..736faff6 100644
--- a/generic/patterns.yml
+++ b/generic/patterns.yml
@@ -15,7 +15,7 @@ patterns:
       additional_not_match:
         # something that means "password" or a placeholder name - either a variable or a placeholder
         # a literal value or config switch
-        - (?i)_?\)?(?:[a-zA-Z0-9._]+[_.])?(?:the )?(?:pass?(wo?r?d|code|phrase)|pass|pwd|secret|token|key|tok|redacted|placeholder|dummy|pw|thephrase|write|read|on|off|true|false|none|value|null( \? )?|nil|undefined|eof|ignore|eol|git|yes|no|y|n|f[0-9]{1,2}|[a-zA-Z]),?\s*\){0,2}[\]>)]?(?:\)\s*\{)?\\?(( or | \|\| ).*)?$
+        - (?i)_?\)?(([a-zA-Z0-9._]+[_.])?(?:the )?(?:pass?(wo?r?d|code|phrase)|pass|pwd|secret|token|key|tok|pw)|redacted|placeholder|dummy|thephrase|write|read|on|off|true|false|none|value|null( \? )?|nil|undefined|eof|ignore|eol|git|yes|no|y|n|f[0-9]{1,2}|[a-zA-Z]),?\s*\){0,2}[\]>)]?(?:\)\s*\{)?\\?(( or | \|\| ).*)?$
         # Python type hints, Swift typing
         - ^\s*(?:(?:typing\.)?(?:(?:[Tt]uple|[Ll]ist|[Dd]ict|Callable|Iterable|Sequence|Optional|Union)\[.*|(?:int|str|float|(?:typing.)?Any|None|bytes|bool|ReadableBuffer)\s*(?:[,|].*)?|(?:Int|Swift\.Int|Int32)\.*))\s*$
         # ..., \, , \n, \0, ',' and other single chars, smilies, hex, digits, nothing at all,

From aad90f5a6c99e69d68b4eb153fa516023f21ce84 Mon Sep 17 00:00:00 2001
From: aegilops <41705651+aegilops@users.noreply.github.com>
Date: Thu, 11 Jul 2024 17:19:13 +0100
Subject: [PATCH 91/95] Fixed generic password additional matches

---
 generic/patterns.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/generic/patterns.yml b/generic/patterns.yml
index 736faff6..5833da6a 100644
--- a/generic/patterns.yml
+++ b/generic/patterns.yml
@@ -15,7 +15,7 @@ patterns:
       additional_not_match:
         # something that means "password" or a placeholder name - either a variable or a placeholder
         # a literal value or config switch
-        - (?i)_?\)?(([a-zA-Z0-9._]+[_.])?(?:the )?(?:pass?(wo?r?d|code|phrase)|pass|pwd|secret|token|key|tok|pw)|redacted|placeholder|dummy|thephrase|write|read|on|off|true|false|none|value|null( \? )?|nil|undefined|eof|ignore|eol|git|yes|no|y|n|f[0-9]{1,2}|[a-zA-Z]),?\s*\){0,2}[\]>)]?(?:\)\s*\{)?\\?(( or | \|\| ).*)?$
+        - ^(?i)_?\)?((a-zA-Z0-9._]+[_.])?(?:the )?(?:pass?(wo?r?d|code|phrase)|pass|pwd|secret|token|key|tok|pw)|redacted|placeholder|dummy|thephrase|write|read|on|off|true|false|none|value|null( \? )?|nil|undefined|eof|ignore|eol|git|yes|no|y|n|f[0-9]{1,2}|[a-zA-Z]),?\s*\){0,2}[\]>)]?(?:\)\s*\{)?\\?(( or | \|\| ).*)?$
         # Python type hints, Swift typing
         - ^\s*(?:(?:typing\.)?(?:(?:[Tt]uple|[Ll]ist|[Dd]ict|Callable|Iterable|Sequence|Optional|Union)\[.*|(?:int|str|float|(?:typing.)?Any|None|bytes|bool|ReadableBuffer)\s*(?:[,|].*)?|(?:Int|Swift\.Int|Int32)\.*))\s*$
         # ..., \, , \n, \0, ',' and other single chars, smilies, hex, digits, nothing at all,

From 859eb6fb79f1c2bbc0d7c041fddec5644fe9af03 Mon Sep 17 00:00:00 2001
From: aegilops <41705651+aegilops@users.noreply.github.com>
Date: Thu, 11 Jul 2024 17:59:39 +0100
Subject: [PATCH 92/95] Tightening generic PW matches more

---
 configs/patterns.yml | 7 +++----
 generic/patterns.yml | 8 +++++---
 2 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/configs/patterns.yml b/configs/patterns.yml
index 79d0fdf1..26b1d095 100644
--- a/configs/patterns.yml
+++ b/configs/patterns.yml
@@ -100,7 +100,7 @@ patterns:
         - ^(keyPassphrase|password|key|[ \t]+|\$\{[^}]+}|(str|string|int|bool)( +#.*)?),?$
         - ^(.* = )?(None|[Tt]rue|[Ff]alse|[Nn]ull|Default(Type)?|Event|[A-Z]+_KEY|VERSION|NAME|update|destroy|(?:dis|en)ableEventListeners|\.\.\.),?$
         - ^(((this|self|obj)\.)([A-Za-z_]+\,|[A-Za-z_].*)|\{\}|\[\]|[0-9a-zA-Z],|\{)$
-        - ^(([a-zA-Z_]+(\(\))?\.)*[a-zA-Z_]+\(\)|\|\s*)$|\{\{[^}]+\}\}|\$\{\{ |^!Ref |^\$\(.*\)|^(https?|file)://|^/[a-zA-Z0-9.-/]+/[a-zA-Z0-9]{5,}\.(pem|crt|key)$|^=|\(\) => |\([^)]+\) => \{
+        - ^(([a-zA-Z_]+(\(\))?\.)*[a-zA-Z_]+\(\)|\|\s*)$|\{\{[^}]+\}\}|\$\{\{ |^!Ref |^\$\(.*\)|^(https?|file)://|^/[a-zA-Z0-9./_-]+/[a-zA-Z0-9_-]{5,}(\.(pem|crt|key|cer|pub|der)|_rsa)$|^=|\(\) => |\([^)]+\) => \{
         - ^\s*(typing\.)?([Tt]uple|[Ll]ist|[Dd]ict|Callable|Iterable|Sequence|Optional|Union)\[.*$
     comments:
       - "Expect large numbers of false positives on variables containing 'key' or 'token'"
@@ -138,7 +138,7 @@ patterns:
         - ^(keyPassphrase|password|key|[ \t]+|\$\{[^}]+}|(str|string|int|bool)( +#.*)?),?$
         - ^(.* = )?(None|[Tt]rue|[Ff]alse|[Nn]ull|Default(Type)?|Event|[A-Z]+_KEY|VERSION|NAME|update|destroy|(?:dis|en)ableEventListeners|\.\.\.),?$
         - ^(((this|self|obj)\.)([A-Za-z_]+\,|[A-Za-z_].*)|\{\}|\[\]|[0-9a-zA-Z],|\{)$
-        - ^(([a-zA-Z_]+(\(\))?\.)*[a-zA-Z_]+\(\)|\|\s*)$|\{\{[^}]+\}\}|\$\{\{ |^!Ref |^\$\(.*\)|^(https?|file)://|^/[a-zA-Z0-9.-/]+/[a-zA-Z0-9]{5,}\.(pem|crt|key)$|^=|\(\) => |\([^)]+\) => \{
+        - ^(([a-zA-Z_]+(\(\))?\.)*[a-zA-Z_]+\(\)|\|\s*)$|\{\{[^}]+\}\}|\$\{\{ |^!Ref |^\$\(.*\)|^(https?|file)://|^/[a-zA-Z0-9./_-]+/[a-zA-Z0-9_-]{5,}(\.(pem|crt|key|cer|pub|der)|_rsa)$|^=|\(\) => |\([^)]+\) => \{
         - ^\s*(typing\.)?([Tt]uple|[Ll]ist|[Dd]ict|Callable|Iterable|Sequence|Optional|Union)\[.*$
     comments:
       - "The hardcoded password is any length"
@@ -200,7 +200,6 @@ patterns:
     test:
       data: |
         password: "password123"
-
       start_offset: 11
       end_offset: 22
 
@@ -328,7 +327,7 @@ patterns:
         # non-secret related content
         - ^(?i)(true|false|y(es)?|no?|on|off|0|1|nill|null|none|(\\x[a-f0-9]{2})+)$
         # a path
-        - '^(/|file:///|[A-Za-z]:/)[A-Za-z0-9._-]{3,}+(/[a-z._-]{1,}){2,}/?$'
+        - '^(/|file:///|https?://[A-Za-z]:/)[A-Za-z0-9._-]{3,}+(/[a-z._-]{1,}){2,}/?$'
     comments:
       - "Looks for secrets in the format of `SECRET=secret` at the start of a line, possibly with an `ENV ` or `export ` prefix"
       - "Allows no whitespace in the secret, to cut false positives"
diff --git a/generic/patterns.yml b/generic/patterns.yml
index 5833da6a..5b4b38a1 100644
--- a/generic/patterns.yml
+++ b/generic/patterns.yml
@@ -21,7 +21,7 @@ patterns:
         # ..., \, , \n, \0, ',' and other single chars, smilies, hex, digits, nothing at all,
         # directories, regex, format string placeholder, urllib demo passphrase, "optional" in docs, a variable substitution, or surrounded by brackets of various kinds
         # all with possible ',' and surrounding whitespace, possibly with a following comment
-        - ^\s*(?:\.\.\.|\\|\\n|\\0|\?|\$\(|[,()[\]{}`.]\\?|-[)(]|\\f21b|0x[A-Fa-f0-9]+|[0-9]{1,4}|(?:~|/tmp|\.\.|\.|/[a-zA-Z/.-]+/[a-zA-Z0-9]+\.(pem|crt|key))|\\{1,2}w\+/g,( \\?)?|%[sr]|geheim\$parole|\([Oo]ptional\).*|\$?(?:\{\{?[^}]+\}\}?|\(\(?[^)]+\)\)?|\[\[?[^\]+]\]\]?)|(before|hover|focus)(,| \{))?,?\s*(?:\s*(?:/\*|#|//).*)?$
+        - ^\s*(?:\.\.\.|\\|\\n|\\0|\?|\$\(|[,()[\]{}`.]\\?|-[)(]|\\f21b|0x[A-Fa-f0-9]+|[0-9]{1,4}|(?:~|/tmp|\.\.|\.|/[a-zA-Z/.-]+/[a-zA-Z0-9]+(\.(pem|crt|key|cer|pub|der)|_rsa))|\\{1,2}w\+/g,( \\?)?|%[sr]|geheim\$parole|\([Oo]ptional\).*|\$?(?:\{\{?[^}]+\}\}?|\(\(?[^)]+\)\)?|\[\[?[^\]+]\]\]?)|(before|hover|focus)(,| \{))?,?\s*(?:\s*(?:/\*|#|//).*)?$
         # function definitions, e.g. Javascript, function calls or variable declaration
         - ^(?:function\s*\([^)]*\)\s*{\s*.*|\([^)]*\)\s*=>\s*(?:{\s*|[^;)]+[;)])|(?:new |\([A-Za-z]+\)\s*)?[a-zA-Z0-9_.]+\s*\(.*|(?:public|private) [A-Za-z0-9_]+ \{|[A-Za-z0-9_.-]+\s*\) \{)$|\{\{[^}]+\}\}|\$\{\{|\{\}$|\[\]$|(0x)?%[0-9]+x|%[dusx]\.$
         # reference to a member variable, index into a variable, bash variables, perl hash key index, environment vars
@@ -57,12 +57,14 @@ patterns:
         \z|[^A-Za-z0-9]
       additional_not_match:
         - |-
-          ^[A-Za-z0-9_.-]*(key|KEY|[Tt]oken|TOKEN)(_[a-zA-Z]+)?['"]?\s*([:=]|=>)\s*["']([Ee]mploye[er]|[Ss]taff|([Ss]earch)?[Rr]esult|[a-z][a-zA-Z]+CSX[A-Z][A-Za-z]+|[A-Za-z]*[Bb]ase(64|32|58)|object|claret|assigns?|clean|contains|error|expand|generate|hoist|indent(ation)?|invert|jumps?|pairs?|param(eter)?s?|pop|rewrite|temp(orary)?|token(s|i[sz]e)?|type|((un)?(quote|shift|wrap|finished))|[a-z]{2,10}([A-Z][a-z]{1,15}){1,6}|(compile|is|has|make|add|each|check|close|cache|format|tag|get|set)([A-Z][A-Za-z]+)?|gadget|classic|(try_)?(base|mode|grade|model)|words|identifier|[a-z.-]+\.(jpe?g|(x|ht)ml|txt|docx?|xlsx?|pdf|png)|enabled|name\.invalidPattern|\.|\.data-api|expect|file|config|ansi|Default(Type)?|Cache-Control|((notD|d)eepE|e)qual|name|NAME|package|version|VERSION|start|end|step|async|Event|throws|ok|notOK|verbose|push(Result)?|slimAssertions|(p|notP)ropEqual|((notS|s)trict|not)Equal|value|prev|next|year|key[0-9]?|destroy|[a-z]+EventListeners|timeout|str(ing)?|hmac|uuid|update|find|true|false|[Nn][ui]ll|[Nn]one|[a-z_]+\.((tf)?state|id|key)|(hibernate|ws|err|i18n|employee|bs|org|com|sun|java)(\.[a-zA-Z0-9_]+){1,4}|[A-Z_]+_KEY)["']$
+          ^[A-Za-z0-9_.-]*(key|KEY|[Tt]oken|TOKEN)(_[a-zA-Z]+)?['"]?\s*([:=]|=>)\s*["']([Ee]mploye[er]|[Ss]taff|([Ss]earch)?[Rr]esult|[a-z][a-zA-Z]+CSX[A-Z][A-Za-z]+|[A-Za-z]*[Bb]ase(64|32|58)|object|claret|assigns?|clean|contains|error|expand|generate|hoist|indent(ation)?|invert|jumps?|pairs?|param(eter)?s?|pop|rewrite|temp(orary)?|token(s|i[sz]e)?|type|((un)?(quote|shift|wrap|finished))|[a-z]{2,10}([A-Z][a-z]{1,15}){1,6}|(compile|is|has|make|add|each|check|close|cache|format|tag|get|set)([A-Z][A-Za-z]+)?|gadget|classic|(try_)?(base|mode|grade|model)|words|identifier|[a-z.-]+\.(jpe?g|(x|ht)ml|txt|docx?|xlsx?|pdf|png)|enabled|name\.invalidPattern|\.|\.data-api|expect|file|config|ansi|Default(Type)?|Cache-Control|((notD|d)eepE|e)qual|name|NAME|package|version|VERSION|start|end|step|async|Event|throws|ok|notOK|verbose|push(Result)?|slimAssertions|(p|notP)ropEqual|((notS|s)trict|not)Equal|value|prev|next|year|key[0-9]?|destroy|[a-z]+EventListeners|timeout|str(ing)?|hmac|uuid|update|find|true|false|val|VAL|REDACTED|redacted|nop|F[0-9]{1,2}|[A-Za-z0-9]|[Nn][ui]ll|[Nn]one|[a-z_]+\.((tf)?state|id|key)|(hibernate|ws|err|i18n|employee|bs|org|com|sun|java)(\.[a-zA-Z0-9_]+){1,4}|[A-Z_]+_KEY)["']$
         - |-
           (?i)(token|key)[_-](name|format|type|enabled|success|type|method)\b
         - |-
-          ^token(_[A-Z]+)?['"]?\s*[:=]\s*['"](barline|parenthesis|qualified|suport|symbol|statementEnd|singleLineTitle|character|pageBreak|operator|optionalTitle|option|zupfnoter|chordname|macro|error|escape|indent|term|titleUnderline|tag|link|literal|(other|table)Block|list|value|control|set|support|injections|array|doc|source|heading|tokens|storage|empty|newline|empty_line|keyword|comment|meta|[lr]?paren|class|punctuation|regexp?|constant|string|entity|invalid|support|variable|multiline|language|paren|markup|singleline|nospell|text|array|doc|source|heading|tokens)(\.{1,2}[A-Za-z0-9_-]+){0,6}[!.]?["']$
+          ^(?i)token(_[A-Z]+)?['"]?\s*[:=]\s*['"](barline|parenthesis|qualified|suport|symbol|statementEnd|singleLineTitle|character|pageBreak|operator|optionalTitle|option|zupfnoter|chordname|macro|error|escape|indent|term|titleUnderline|tag|link|literal|(other|table)Block|list|value|control|set|support|injections|array|doc|source|heading|tokens|storage|empty|newline|empty_line|keyword|(line)?comment|meta|[lr]?paren|class|punctuation|regexp?|constant|string|entity|invalid|support|variable|multiline|language|paren|markup|singleline|nospell|text|array|doc|source|heading|tokens)(\.{1,2}[A-Za-z0-9_-]+){0,6}[!.]?["']$
         - "^KEY_[A-Z]+[0-9]{0,3}: 'k[a-zA-Z0-9]{1,6}'$"
+        - |-
+          ['"` ](/dev/u?random|(^/[a-zA-Z0-9./_-]+/[a-zA-Z0-9_-]{5,}(\.(pem|crt|key|cer|pub|der)|_rsa)|https?://.*|file://.*)['"`]$
     test:
       data: password="Password123"
       start_offset: 0

From 4c20ffb7a806b13e0bb1b3b48034d6fa5c87a4ea Mon Sep 17 00:00:00 2001
From: aegilops <41705651+aegilops@users.noreply.github.com>
Date: Thu, 11 Jul 2024 18:04:01 +0100
Subject: [PATCH 93/95] Fix missing bracket issue

---
 configs/patterns.yml | 4 ++--
 generic/patterns.yml | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/configs/patterns.yml b/configs/patterns.yml
index 26b1d095..8e347453 100644
--- a/configs/patterns.yml
+++ b/configs/patterns.yml
@@ -100,7 +100,7 @@ patterns:
         - ^(keyPassphrase|password|key|[ \t]+|\$\{[^}]+}|(str|string|int|bool)( +#.*)?),?$
         - ^(.* = )?(None|[Tt]rue|[Ff]alse|[Nn]ull|Default(Type)?|Event|[A-Z]+_KEY|VERSION|NAME|update|destroy|(?:dis|en)ableEventListeners|\.\.\.),?$
         - ^(((this|self|obj)\.)([A-Za-z_]+\,|[A-Za-z_].*)|\{\}|\[\]|[0-9a-zA-Z],|\{)$
-        - ^(([a-zA-Z_]+(\(\))?\.)*[a-zA-Z_]+\(\)|\|\s*)$|\{\{[^}]+\}\}|\$\{\{ |^!Ref |^\$\(.*\)|^(https?|file)://|^/[a-zA-Z0-9./_-]+/[a-zA-Z0-9_-]{5,}(\.(pem|crt|key|cer|pub|der)|_rsa)$|^=|\(\) => |\([^)]+\) => \{
+        - ^(([a-zA-Z_]+(\(\))?\.)*[a-zA-Z_]+\(\)|\|\s*)$|\{\{[^}]+\}\}|\$\{\{ |^!Ref |^\$\(.*\)|^(https?|file)://|^(/[a-zA-Z0-9./_-]+/)?[a-zA-Z0-9_-]{5,}(\.(pem|crt|key|cer|pub|der)|_rsa)$|^=|\(\) => |\([^)]+\) => \{
         - ^\s*(typing\.)?([Tt]uple|[Ll]ist|[Dd]ict|Callable|Iterable|Sequence|Optional|Union)\[.*$
     comments:
       - "Expect large numbers of false positives on variables containing 'key' or 'token'"
@@ -138,7 +138,7 @@ patterns:
         - ^(keyPassphrase|password|key|[ \t]+|\$\{[^}]+}|(str|string|int|bool)( +#.*)?),?$
         - ^(.* = )?(None|[Tt]rue|[Ff]alse|[Nn]ull|Default(Type)?|Event|[A-Z]+_KEY|VERSION|NAME|update|destroy|(?:dis|en)ableEventListeners|\.\.\.),?$
         - ^(((this|self|obj)\.)([A-Za-z_]+\,|[A-Za-z_].*)|\{\}|\[\]|[0-9a-zA-Z],|\{)$
-        - ^(([a-zA-Z_]+(\(\))?\.)*[a-zA-Z_]+\(\)|\|\s*)$|\{\{[^}]+\}\}|\$\{\{ |^!Ref |^\$\(.*\)|^(https?|file)://|^/[a-zA-Z0-9./_-]+/[a-zA-Z0-9_-]{5,}(\.(pem|crt|key|cer|pub|der)|_rsa)$|^=|\(\) => |\([^)]+\) => \{
+        - ^(([a-zA-Z_]+(\(\))?\.)*[a-zA-Z_]+\(\)|\|\s*)$|\{\{[^}]+\}\}|\$\{\{ |^!Ref |^\$\(.*\)|^(https?|file)://|^(/[a-zA-Z0-9./_-]+/)?[a-zA-Z0-9_-]{5,}(\.(pem|crt|key|cer|pub|der)|_rsa)$|^=|\(\) => |\([^)]+\) => \{
         - ^\s*(typing\.)?([Tt]uple|[Ll]ist|[Dd]ict|Callable|Iterable|Sequence|Optional|Union)\[.*$
     comments:
       - "The hardcoded password is any length"
diff --git a/generic/patterns.yml b/generic/patterns.yml
index 5b4b38a1..423ed5bf 100644
--- a/generic/patterns.yml
+++ b/generic/patterns.yml
@@ -21,7 +21,7 @@ patterns:
         # ..., \, , \n, \0, ',' and other single chars, smilies, hex, digits, nothing at all,
         # directories, regex, format string placeholder, urllib demo passphrase, "optional" in docs, a variable substitution, or surrounded by brackets of various kinds
         # all with possible ',' and surrounding whitespace, possibly with a following comment
-        - ^\s*(?:\.\.\.|\\|\\n|\\0|\?|\$\(|[,()[\]{}`.]\\?|-[)(]|\\f21b|0x[A-Fa-f0-9]+|[0-9]{1,4}|(?:~|/tmp|\.\.|\.|/[a-zA-Z/.-]+/[a-zA-Z0-9]+(\.(pem|crt|key|cer|pub|der)|_rsa))|\\{1,2}w\+/g,( \\?)?|%[sr]|geheim\$parole|\([Oo]ptional\).*|\$?(?:\{\{?[^}]+\}\}?|\(\(?[^)]+\)\)?|\[\[?[^\]+]\]\]?)|(before|hover|focus)(,| \{))?,?\s*(?:\s*(?:/\*|#|//).*)?$
+        - ^\s*(?:\.\.\.|\\|\\n|\\0|\?|\$\(|[,()[\]{}`.]\\?|-[)(]|\\f21b|0x[A-Fa-f0-9]+|[0-9]{1,4}|(?:~|/tmp|\.\.|\.|(/[a-zA-Z0-9./_-]+/)?[a-zA-Z0-9]+(\.(pem|crt|key|cer|pub|der)|_rsa))|\\{1,2}w\+/g,( \\?)?|%[sr]|geheim\$parole|\([Oo]ptional\).*|\$?(?:\{\{?[^}]+\}\}?|\(\(?[^)]+\)\)?|\[\[?[^\]+]\]\]?)|(before|hover|focus)(,| \{))?,?\s*(?:\s*(?:/\*|#|//).*)?$
         # function definitions, e.g. Javascript, function calls or variable declaration
         - ^(?:function\s*\([^)]*\)\s*{\s*.*|\([^)]*\)\s*=>\s*(?:{\s*|[^;)]+[;)])|(?:new |\([A-Za-z]+\)\s*)?[a-zA-Z0-9_.]+\s*\(.*|(?:public|private) [A-Za-z0-9_]+ \{|[A-Za-z0-9_.-]+\s*\) \{)$|\{\{[^}]+\}\}|\$\{\{|\{\}$|\[\]$|(0x)?%[0-9]+x|%[dusx]\.$
         # reference to a member variable, index into a variable, bash variables, perl hash key index, environment vars
@@ -64,7 +64,7 @@ patterns:
           ^(?i)token(_[A-Z]+)?['"]?\s*[:=]\s*['"](barline|parenthesis|qualified|suport|symbol|statementEnd|singleLineTitle|character|pageBreak|operator|optionalTitle|option|zupfnoter|chordname|macro|error|escape|indent|term|titleUnderline|tag|link|literal|(other|table)Block|list|value|control|set|support|injections|array|doc|source|heading|tokens|storage|empty|newline|empty_line|keyword|(line)?comment|meta|[lr]?paren|class|punctuation|regexp?|constant|string|entity|invalid|support|variable|multiline|language|paren|markup|singleline|nospell|text|array|doc|source|heading|tokens)(\.{1,2}[A-Za-z0-9_-]+){0,6}[!.]?["']$
         - "^KEY_[A-Z]+[0-9]{0,3}: 'k[a-zA-Z0-9]{1,6}'$"
         - |-
-          ['"` ](/dev/u?random|(^/[a-zA-Z0-9./_-]+/[a-zA-Z0-9_-]{5,}(\.(pem|crt|key|cer|pub|der)|_rsa)|https?://.*|file://.*)['"`]$
+          ['"` ](/dev/u?random|(/[a-zA-Z0-9./_-]+/)?[a-zA-Z0-9_-]{5,}(\.(pem|crt|key|cer|pub|der)|_rsa)|https?://.*|file://.*)['"`]$
     test:
       data: password="Password123"
       start_offset: 0

From 2a485cce45fffaa12b160f72fc2d22348c36e445 Mon Sep 17 00:00:00 2001
From: aegilops <41705651+aegilops@users.noreply.github.com>
Date: Thu, 11 Jul 2024 18:05:43 +0100
Subject: [PATCH 94/95] Updated expected matches

---
 generic/patterns.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/generic/patterns.yml b/generic/patterns.yml
index 423ed5bf..409432d2 100644
--- a/generic/patterns.yml
+++ b/generic/patterns.yml
@@ -37,6 +37,9 @@ patterns:
       - name: passwords.js
         start_offset: 41
         end_offset: 62
+      - name: passwords.js
+        start_offset: 97
+        end_offset: 112
     comments:
       - "Likely to cause large numbers of false positives - use with caution"
       - "`password`, `secret`, `key`, or password like prefix (fuzzy)"

From 2a50bed5e8b4e54916bfbf251c2269406733b6af Mon Sep 17 00:00:00 2001
From: aegilops <41705651+aegilops@users.noreply.github.com>
Date: Thu, 11 Jul 2024 18:07:04 +0100
Subject: [PATCH 95/95] Updated README.md

---
 configs/README.md | 26 ++++++++++++++------------
 generic/README.md | 28 +++++++++++++++-------------
 2 files changed, 29 insertions(+), 25 deletions(-)

diff --git a/configs/README.md b/configs/README.md
index 0dc306fe..5fbbc228 100644
--- a/configs/README.md
+++ b/configs/README.md
@@ -170,27 +170,27 @@ Add these additional matches to the [Secret Scanning Custom Pattern](https://doc
 - Not Match:
 
   ```regex
-  ^(?:keyPassphrase|password|key|[ \t]+|\$\{[^}]+}|(?:str|string|int|bool)( +#.*)?),?$
+  ^(keyPassphrase|password|key|[ \t]+|\$\{[^}]+}|(str|string|int|bool)( +#.*)?),?$
   ```
 - Not Match:
 
   ```regex
-  ^(?:.* = )?(?:None|[Tt]rue|[Ff]alse|[Nn]ull|Default(?:Type)?|Event|[A-Z]+_KEY|VERSION|NAME|update|destroy|(?:dis|en)ableEventListeners|\.\.\.),?$
+  ^(.* = )?(None|[Tt]rue|[Ff]alse|[Nn]ull|Default(Type)?|Event|[A-Z]+_KEY|VERSION|NAME|update|destroy|(?:dis|en)ableEventListeners|\.\.\.),?$
   ```
 - Not Match:
 
   ```regex
-  ^(?:(?:this|self|obj)\.)(?:[A-Za-z_]+\,|[A-Za-z_].*)$
+  ^(((this|self|obj)\.)([A-Za-z_]+\,|[A-Za-z_].*)|\{\}|\[\]|[0-9a-zA-Z],|\{)$
   ```
 - Not Match:
 
   ```regex
-  ^(?:(?:[a-zA-Z_]+(?:\(\))?\.)*[a-zA-Z_]+\(\)|\|\s*)$|\{\{[^}]+\}\}|\$\{\{ |^!Ref
+  ^(([a-zA-Z_]+(\(\))?\.)*[a-zA-Z_]+\(\)|\|\s*)$|\{\{[^}]+\}\}|\$\{\{ |^!Ref |^\$\(.*\)|^(https?|file)://|^(/[a-zA-Z0-9./_-]+/)?[a-zA-Z0-9_-]{5,}(\.(pem|crt|key|cer|pub|der)|_rsa)$|^=|\(\) => |\([^)]+\) => \{
   ```
 - Not Match:
 
   ```regex
-  ^\s*(?:typing\.)?(?:[Tt]uple|[Ll]ist|[Dd]ict|Callable|Iterable|Sequence|Optional|Union)\[.*$
+  ^\s*(typing\.)?([Tt]uple|[Ll]ist|[Dd]ict|Callable|Iterable|Sequence|Optional|Union)\[.*$
   ```
 
 </details>
@@ -248,27 +248,27 @@ Add these additional matches to the [Secret Scanning Custom Pattern](https://doc
 - Not Match:
 
   ```regex
-  ^(?:keyPassphrase|password|key|[ \t]+|\$\{[^}]+}|(?:str|string|int|bool)( +#.*)?),?$
+  ^(keyPassphrase|password|key|[ \t]+|\$\{[^}]+}|(str|string|int|bool)( +#.*)?),?$
   ```
 - Not Match:
 
   ```regex
-  ^(?:.* = )?(?:None|[Tt]rue|[Ff]alse|[Nn]ull|Default(?:Type)?|Event|[A-Z]+_KEY|VERSION|NAME|update|destroy|(?:dis|en)ableEventListeners|\.\.\.),?$
+  ^(.* = )?(None|[Tt]rue|[Ff]alse|[Nn]ull|Default(Type)?|Event|[A-Z]+_KEY|VERSION|NAME|update|destroy|(?:dis|en)ableEventListeners|\.\.\.),?$
   ```
 - Not Match:
 
   ```regex
-  ^(?:(?:(?:this|self|obj)\.)(?:[A-Za-z_]+\,|[A-Za-z_].*)|([!&*{}[\],#|>@`"'%]|[:?-] ).*)$
+  ^(((this|self|obj)\.)([A-Za-z_]+\,|[A-Za-z_].*)|\{\}|\[\]|[0-9a-zA-Z],|\{)$
   ```
 - Not Match:
 
   ```regex
-  ^(?:[a-zA-Z_]+(?:\(\))?\.)*[a-zA-Z_]+\(\)$|\$\{\{[^}]+\}\}
+  ^(([a-zA-Z_]+(\(\))?\.)*[a-zA-Z_]+\(\)|\|\s*)$|\{\{[^}]+\}\}|\$\{\{ |^!Ref |^\$\(.*\)|^(https?|file)://|^(/[a-zA-Z0-9./_-]+/)?[a-zA-Z0-9_-]{5,}(\.(pem|crt|key|cer|pub|der)|_rsa)$|^=|\(\) => |\([^)]+\) => \{
   ```
 - Not Match:
 
   ```regex
-  ^\s*(?:typing\.)?(?:[Tt]uple|[Ll]ist|[Dd]ict|Callable|Iterable|Sequence|Optional|Union)\[.*$
+  ^\s*(typing\.)?([Tt]uple|[Ll]ist|[Dd]ict|Callable|Iterable|Sequence|Optional|Union)\[.*$
   ```
 
 </details>
@@ -581,13 +581,15 @@ _version: v0.1_
 - The pattern only checks for certain key words to begin the pattern (`secret`, `password`, etc.)
 
 - More restrictive than the Generic Passwords pattern, so less prone to false positives
+
+- Lower length limit of 8 to remove FPs
   
 
 <details>
 <summary>Pattern Format</summary>
 
 ```regex
-[^\r\n\x00-\x08'"#]+
+[^\r\n\x00-\x08'"#]{8,}
 ```
 
 </details>
@@ -637,7 +639,7 @@ Add these additional matches to the [Secret Scanning Custom Pattern](https://doc
 - Not Match:
 
   ```regex
-  ^(/|file:///|[A-Za-z]:/)[A-Za-z0-9._-]{3,}+(/[a-z._-]{1,}){2,}/?$
+  ^(/|file:///|https?://[A-Za-z]:/)[A-Za-z0-9._-]{3,}+(/[a-z._-]{1,}){2,}/?$
   ```
 
 </details>
diff --git a/generic/README.md b/generic/README.md
index 35a00819..5061513a 100644
--- a/generic/README.md
+++ b/generic/README.md
@@ -59,7 +59,7 @@ Add these additional matches to the [Secret Scanning Custom Pattern](https://doc
 - Not Match:
 
   ```regex
-  _?\)?(?:[a-zA-Z0-9._]+[_.])?(?:the )?(?:pass?(wo?r?d|code|phrase)|pass|pwd|secret|token|tok|redacted|placeholder|dummy|pw|thephrase|write|read|on|off|true|false|none|null( \? )?|nil|undefined|eof|ignore|eol|git|yes|no|y|n),?\s*\){0,2}[\]>)]?(?:\)\s*\{)?\\?(( or | \|\| ).*)?$
+  ^(?i)_?\)?((a-zA-Z0-9._]+[_.])?(?:the )?(?:pass?(wo?r?d|code|phrase)|pass|pwd|secret|token|key|tok|pw)|redacted|placeholder|dummy|thephrase|write|read|on|off|true|false|none|value|null( \? )?|nil|undefined|eof|ignore|eol|git|yes|no|y|n|f[0-9]{1,2}|[a-zA-Z]),?\s*\){0,2}[\]>)]?(?:\)\s*\{)?\\?(( or | \|\| ).*)?$
   ```
 - Not Match:
 
@@ -69,17 +69,17 @@ Add these additional matches to the [Secret Scanning Custom Pattern](https://doc
 - Not Match:
 
   ```regex
-  ^\s*(?:\.\.\.|\\|\\n|\\0|\?|\$\(|[,()[\]{}`.]\\?|-[)(]|\\f21b|0x[A-Fa-f0-9]+|[0-9]{1,4}|(?:~|/tmp|\.\.|\.)|\\{1,2}w\+/g,( \\?)?|%[sr]|geheim\$parole|\([Oo]ptional\).*|\$?(?:\{\{?[^}]+\}\}?|\(\(?[^)]+\)\)?|\[\[?[^\]+]\]\]?)|(before|hover|focus)(,| \{))?,?\s*(?:\s*(?:/\*|#|//).*)?$
+  ^\s*(?:\.\.\.|\\|\\n|\\0|\?|\$\(|[,()[\]{}`.]\\?|-[)(]|\\f21b|0x[A-Fa-f0-9]+|[0-9]{1,4}|(?:~|/tmp|\.\.|\.|(/[a-zA-Z0-9./_-]+/)?[a-zA-Z0-9]+(\.(pem|crt|key|cer|pub|der)|_rsa))|\\{1,2}w\+/g,( \\?)?|%[sr]|geheim\$parole|\([Oo]ptional\).*|\$?(?:\{\{?[^}]+\}\}?|\(\(?[^)]+\)\)?|\[\[?[^\]+]\]\]?)|(before|hover|focus)(,| \{))?,?\s*(?:\s*(?:/\*|#|//).*)?$
   ```
 - Not Match:
 
   ```regex
-  ^(?:function\s*\([^)]*\)\s*{\s*.*|\([^)]*\)\s*=>\s*(?:{\s*|[^;)]+[;)])|(?:new |\([A-Za-z]+\)\s*)?[a-zA-Z0-9_.]+\s*\(.*|(?:public|private) [A-Za-z0-9_]+ \{|[A-Za-z0-9_.-]+\s*\) \{)$|\{\{[^}]+\}\}|\$\{\{
+  ^(?:function\s*\([^)]*\)\s*{\s*.*|\([^)]*\)\s*=>\s*(?:{\s*|[^;)]+[;)])|(?:new |\([A-Za-z]+\)\s*)?[a-zA-Z0-9_.]+\s*\(.*|(?:public|private) [A-Za-z0-9_]+ \{|[A-Za-z0-9_.-]+\s*\) \{)$|\{\{[^}]+\}\}|\$\{\{|\{\}$|\[\]$|(0x)?%[0-9]+x|%[dusx]\.$
   ```
 - Not Match:
 
   ```regex
-  ^\s*(?:(?:self|this)\.[a-zA-Z_][a-zA-Z0-9_.]+[,[]?|[a-zA-Z0-9_.]+\[(?:[a-zA-Z0-9_.]+)?\]?|\$(?:[1-9]|[A-Za-z0-9_]+)\{?|os\.environ\[[^\]]\]|process\.env\.[A-Z0-9_]+)\s*(?:,|\|\||&&)?\s*$
+  ^\s*(?:(?:self|this)\.[a-zA-Z_][a-zA-Z0-9_.]+[,[]?|[a-zA-Z0-9_.]+\[(?:[a-zA-Z0-9_.]+)?\]?|\$(?:[1-9]|[A-Za-z0-9_]+)\{?|os\.environ\[[^\]]\]|process\.env\.[A-Z0-9_]+)\s*(?:,|\|\||&&)?\s*$|`(%s|\+)
   ```
 
 </details>
@@ -138,7 +138,7 @@ Add these additional matches to the [Secret Scanning Custom Pattern](https://doc
 - Not Match:
 
   ```regex
-  ^[A-Za-z0-9_.-]*(key|KEY|[Tt]oken|TOKEN)(_[a-zA-Z]+)?['"]?\s*([:=]|=>)\s*["']([Ee]mploye[er]|[Ss]taff|([Ss]earch)?[Rr]esult|[a-z][a-zA-Z]+CSX[A-Z][A-Za-z]+|[A-Za-z]*[Bb]ase(64|32|58)|object|claret|assigns?|clean|contains|error|expand|generate|hoist|indent(ation)?|invert|jumps?|pairs?|param(eter)?s?|pop|rewrite|temp(orary)?|token(s|i[sz]e)?|type|((un)?(quote|shift|wrap|finished))|[a-z]{2,10}([A-Z][a-z]{1,15}){1,6}|(compile|is|has|make|add|each|check|close|cache|format|tag|get|set)([A-Z][A-Za-z]+)?|gadget|classic|(try_)?(base|mode|grade|model)|words|identifier|[a-z.-]+\.(jpe?g|(x|ht)ml|txt|docx?|xlsx?|pdf|png)|enabled|name\.invalidPattern|\.|\.data-api|expect|file|config|ansi|Default(Type)?|Cache-Control|((notD|d)eepE|e)qual|name|NAME|package|version|VERSION|start|end|step|async|Event|throws|ok|notOK|verbose|push(Result)?|slimAssertions|(p|notP)ropEqual|((notS|s)trict|not)Equal|value|prev|next|year|key[0-9]?|destroy|[a-z]+EventListeners|timeout|str(ing)?|hmac|uuid|update|find|true|false|[Nn][ui]ll|[Nn]one|[a-z_]+\.((tf)?state|id|key)|(hibernate|ws|err|i18n|employee|bs|org|com|sun|java)(\.[a-zA-Z0-9_]+){1,4}|[A-Z_]+_KEY)["']$
+  ^[A-Za-z0-9_.-]*(key|KEY|[Tt]oken|TOKEN)(_[a-zA-Z]+)?['"]?\s*([:=]|=>)\s*["']([Ee]mploye[er]|[Ss]taff|([Ss]earch)?[Rr]esult|[a-z][a-zA-Z]+CSX[A-Z][A-Za-z]+|[A-Za-z]*[Bb]ase(64|32|58)|object|claret|assigns?|clean|contains|error|expand|generate|hoist|indent(ation)?|invert|jumps?|pairs?|param(eter)?s?|pop|rewrite|temp(orary)?|token(s|i[sz]e)?|type|((un)?(quote|shift|wrap|finished))|[a-z]{2,10}([A-Z][a-z]{1,15}){1,6}|(compile|is|has|make|add|each|check|close|cache|format|tag|get|set)([A-Z][A-Za-z]+)?|gadget|classic|(try_)?(base|mode|grade|model)|words|identifier|[a-z.-]+\.(jpe?g|(x|ht)ml|txt|docx?|xlsx?|pdf|png)|enabled|name\.invalidPattern|\.|\.data-api|expect|file|config|ansi|Default(Type)?|Cache-Control|((notD|d)eepE|e)qual|name|NAME|package|version|VERSION|start|end|step|async|Event|throws|ok|notOK|verbose|push(Result)?|slimAssertions|(p|notP)ropEqual|((notS|s)trict|not)Equal|value|prev|next|year|key[0-9]?|destroy|[a-z]+EventListeners|timeout|str(ing)?|hmac|uuid|update|find|true|false|val|VAL|REDACTED|redacted|nop|F[0-9]{1,2}|[A-Za-z0-9]|[Nn][ui]ll|[Nn]one|[a-z_]+\.((tf)?state|id|key)|(hibernate|ws|err|i18n|employee|bs|org|com|sun|java)(\.[a-zA-Z0-9_]+){1,4}|[A-Z_]+_KEY)["']$
   ```
 - Not Match:
 
@@ -148,13 +148,18 @@ Add these additional matches to the [Secret Scanning Custom Pattern](https://doc
 - Not Match:
 
   ```regex
-  ^token(_[A-Z]+)?['"]?\s*[:=]\s*['"](barline|parenthesis|qualified|suport|symbol|statementEnd|singleLineTitle|character|pageBreak|operator|optionalTitle|option|zupfnoter|chordname|macro|error|escape|indent|term|titleUnderline|tag|link|literal|(other|table)Block|list|value|control|set|support|injections|array|doc|source|heading|tokens|storage|empty|newline|empty_line|keyword|comment|meta|[lr]?paren|class|punctuation|regexp?|constant|string|entity|invalid|support|variable|multiline|language|paren|markup|singleline|nospell|text|array|doc|source|heading|tokens)(\.{1,2}[A-Za-z0-9_-]+){0,6}[!.]?["']$
+  ^(?i)token(_[A-Z]+)?['"]?\s*[:=]\s*['"](barline|parenthesis|qualified|suport|symbol|statementEnd|singleLineTitle|character|pageBreak|operator|optionalTitle|option|zupfnoter|chordname|macro|error|escape|indent|term|titleUnderline|tag|link|literal|(other|table)Block|list|value|control|set|support|injections|array|doc|source|heading|tokens|storage|empty|newline|empty_line|keyword|(line)?comment|meta|[lr]?paren|class|punctuation|regexp?|constant|string|entity|invalid|support|variable|multiline|language|paren|markup|singleline|nospell|text|array|doc|source|heading|tokens)(\.{1,2}[A-Za-z0-9_-]+){0,6}[!.]?["']$
   ```
 - Not Match:
 
   ```regex
   ^KEY_[A-Z]+[0-9]{0,3}: 'k[a-zA-Z0-9]{1,6}'$
   ```
+- Not Match:
+
+  ```regex
+  ['"` ](/dev/u?random|(/[a-zA-Z0-9./_-]+/)?[a-zA-Z0-9_-]{5,}(\.(pem|crt|key|cer|pub|der)|_rsa)|https?://.*|file://.*)['"`]$
+  ```
 
 </details>
 
@@ -423,13 +428,15 @@ _version: v0.2_
 - As used in an Authorization header
 
 - We try to remove common placeholders
+
+- Lower length limit of 12 to remove common false positives on "Token <common word>", since most words are below 12 characters in length
   
 
 <details>
 <summary>Pattern Format</summary>
 
 ```regex
-[a-zA-Z0-9_.=/+:-]+
+[a-zA-Z0-9_.=/+:-]{12,}
 ```
 
 </details>
@@ -474,12 +481,7 @@ Add these additional matches to the [Secret Scanning Custom Pattern](https://doc
 - Not Match:
 
   ```regex
-  (?i)x{5}
-  ```
-- Not Match:
-
-  ```regex
-  ^(?i)(x+|y+|z+|a+|\.+|.*\.\.\.)$
+  (^(?i)(x+|y+|z+|a+|\.+|.*\.\.\.)$|(?i)x{5})
   ```
 
 </details>