Restore expanded redaction as a default feature. Add a set of new, hi…

…ghly accurate detections.
microsoft · Oct 12, 2023 · 6e7b55a · 6e7b55a
1 parent 81514c8
commit 6e7b55a
Show file tree

Hide file tree

Showing 3 changed files with 65 additions and 172 deletions.
diff --git a/src/Agent.Sdk/Knob/AgentKnobs.cs b/src/Agent.Sdk/Knob/AgentKnobs.cs
@@ -288,7 +288,7 @@ public class AgentKnobs
             nameof(MaskUsingCredScanRegexes),
             "Use the CredScan regexes for masking secrets. CredScan is an internal tool developed at Microsoft to keep passwords and authentication keys from being checked in. This defaults to disabled, as there are performance problems with some task outputs.",
             new EnvironmentKnobSource("AZP_USE_CREDSCAN_REGEXES"),
-            new BuiltInDefaultKnobSource("false"));
+            new BuiltInDefaultKnobSource("true"));
 
         public static readonly Knob MaskedSecretMinLength = new Knob(
             nameof(MaskedSecretMinLength),

diff --git a/src/Microsoft.VisualStudio.Services.Agent/AdditionalMaskingRegexes.CredScan.cs b/src/Microsoft.VisualStudio.Services.Agent/AdditionalMaskingRegexes.CredScan.cs
@@ -30,176 +30,55 @@ public static partial class AdditionalMaskingRegexes
         private static IEnumerable<string> credScanPatterns =
             new List<string>()
             {
-                // AnsibleVaultData
-                  @"" // pre-match
-                + @"\$ANSIBLE_VAULT;\d+\.\d+;AES256\s+\d+" // match
+                // AAD client app, most recent two versions.
+                  @"\b" // pre-match
+                + @"[0-9A-Za-z-_~.]{3}7Q~[0-9A-Za-z-_~.]{31}\b|\b[0-9A-Za-z-_~.]{3}8Q~[0-9A-Za-z-_~.]{34}" // match
+                + @"\b", // post-match
+
+                  // Prominent Azure provider 512-bit symmetric keys.
+                  @"\b" // pre-match
+                + @"[0-9A-Za-z+/]{76}(APIM|ACDb|\+(ABa|AMC|ASt))[0-9A-Za-z+/]{5}[AQgw]==" // match
                 + @"", // post-match
-
-                // AzurePowerShellTokenCache
-                  @"" // pre-match
-                + @"[""']TokenCache[""']\s*:\s*\{\s*[""']CacheData[""']\s*:\s*[""'][a-z0-9/\+]{86}" // match
-                + @"", // post-match
-
-                // Base64EncodedStringLiteral
-                  @"(?<=(^|[""'>;=\s#]))" // pre-match
-                + @"(?<DataBlock>(?-i)MI(?i)[a-z0-9/+\s\u0085\u200b""',\\]{200,20000}[a-z0-9/+]={0,2})" // match
+                       // 
+                  // Prominent Azure provider 256-bit symmetric keys.
+                  @"\b" // pre-match
+                + @"[0-9A-Za-z+/]{33}(AIoT|\+(ASb|AEh|ARm))[A-P][0-9A-Za-z+/]{5}=" // match
                 + @"", // post-match
-
-                // JsonWebToken
-                  @"" // pre-match
-                + @"(?-i)(?<JwtToken>eyJ(?i)[a-z0-9\-_%]+\.(?-i)eyJ(?i)[a-z0-9\-_%]+\.[a-z0-9\-_%]+)|([rR]efresh_?[tT]oken|REFRESH_?TOKEN)[""']?\s*[:=]{1,2}\s*[""']?(?<JwtToken>(\w+-)+\w+)[""']?" // match
-                + @"", // post-match
-
-                // SlackTokens
-                  @"" // pre-match
-                + @"xox[pbar]\-[a-z0-9\-]+" // match
-                + @"", // post-match
-
-                // SymmetricKey128
-                  @"(?<=[^\w/\+\._\$,\\])" // pre-match
-                + @"(?<SymmetricKey>[a-z0-9/\+]{22}==)" // match
-                + @"(?=([^\w/\+\.\$]|$))", // post-match
-
-                // SymmetricKey128Hex
-                  @"(?<=[^\w/\+\._\$,\\][dapi]+)" // pre-match
-                + @"(?<SymmetricKey>[a-f0-9]{32})" // match
-                + @"(?=([^\w/\+\.\$]|$))", // post-match
-
-                // SymmetricKey160Hex
-                  @"(?<=[^\w/\+\._\$,\\])" // pre-match
-                + @"(?<Hex160>[a-f0-9/\+]{40})" // match
-                + @"(?=([^\w/\+\.\$]|$))", // post-match
-
-                // SymmetricKey232
-                  @"(?<=[^\w/\+\._\$,\\])" // pre-match
-                + @"(?<SymmetricKey>(?-i)AIza(?i)[a-z0-9_\\\-]{35})" // match
-                + @"(?=([^\w/\+\.\$]|$))", // post-match
-
-                // SymmetricKey240
-                  @"(?<=[^\w/\+\.\-\$,\\])" // pre-match
-                + @"(?<SymmetricKey>[a-z0-9/\+]{40})" // match
-                + @"(?=([^\w/\+\.\-\$,\\]|$))", // post-match
-
-                // SymmetricKey256
-                  @"(?<=[^\w/\+\.\$,\\])" // pre-match
-                + @"(?<SymmetricKey>[a-z0-9/\+]{43}=)" // match
-                + @"(?=([^\w/\+\.\$]|$))", // post-match
-
-                // SymmetricKey256B32
-                  @"(?<=[^\w/\+\._\-\$,\\])" // pre-match
-                + @"(?<SymmetricKey>(?-i)[a-z2-7]{52}(?i))" // match
-                + @"(?=(?<=[0-9]+[a-z]+[0-9]+.{0,49})([^\w/\+\.\-\$,]|$))", // post-match
-
-                // SymmetricKey256UrlEncoded
-                  @"(?<=[^\w/\+\._\-\$,\\%])" // pre-match
-                + @"(?<SymmetricKey>[a-z0-9%]{43,63}%3d)" // match
+
+                  // Azure Function key.
+                  @"\b" // pre-match
+                + @"[0-9A-Za-z_\-]{44}AzFu[0-9A-Za-z\-_]{5}[AQgw]==" // match
                 + @"", // post-match
 
-                // SymmetricKey320
-                  @"(?<=[^\w/\+\.\-\$,\\])" // pre-match
-                + @"(?<SymmetricKey>[a-z0-9/\+]{54}={2})" // match
-                + @"(?=([^\w/\+\.\-\$,\\]|$))", // post-match
-
-                // SymmetricKey320UrlEncoded
-                  @"(?<=[^\w/\+\.\-\$,\\%])" // pre-match
-                + @"(?<SymmetricKey>[a-z0-9%]{54,74}(%3d){2})" // match
+                  // Azure Search query and admin keys.
+                  @"\b" // pre-match
+                + @"[0-9A-Za-z]{42}AzSe[A-D][0-9A-Za-z]{5}" // match
+                + @"\b", // post-match
+
+                  // Azure Container Registry keys.
+                  @"\b" // pre-match
+                + @"[0-9A-Za-z+/]{42}\+ACR[A-D][0-9A-Za-z+/]{5}" // match
+                + @"\b", // post-match
+
+                  // Azure Cache for Redis keys.
+                  @"\b" // pre-match
+                + @"[0-9A-Za-z]{33}AzCa[A-P][0-9A-Za-z]{5}=" // match
                 + @"", // post-match
-
-                // SymmetricKey360
-                  @"(?<=[^\w/\+\.\-\$,\\])" // pre-match
-                + @"(?<SymmetricKey>[a-z0-9/\+]{60})" // match
-                + @"(?=[^\w/\+\.\-\$,\\])", // post-match
-
-                // SymmetricKey512
-                  @"(?<=[^\r\n\t\w/\+\.\-\$,\\])" // pre-match
-                + @"(?<SymmetricKey>[a-z0-9/\+]{86}==)" // match
-                + @"(?=([^\w/\+\.\-\$]|$))", // post-match
-
-                // AzureActiveDirectoryLoginCredentials
-                  @"(?<=@([a-z0-9.]+\.(on)?)?microsoft\.com[ -~\s\u200b]{1,80}?(userpass|password|pwd|pw|\wpass[ =:>]+|(get|make)securestring)\W)" // pre-match
-                + @"(?<Password>[^\s;`,""'\(\)]{10,80})" // match
-                + @"(?=[\s;`,""'\(\)])", // post-match
-
-                // AzureActiveDirectoryLoginCredentials
-                  @"" // pre-match
-                + @"(?<MigrationPassword>(\-destinationPasswordPlain ""[^""]+?""))" // match
-                + @"(?=[ -~\s\u200b]{1,150}?@([a-z0-9.]+\.(on)?)?microsoft\.com)", // post-match
-
-                // AzureActiveDirectoryLoginCredentials
-                  @"(?<=(sign_in|SharePointOnlineAuthenticatedContext|(User|Exchange)Credentials?|password)[ -~\s\u200b]{1,100}?@([a-z0-9.]+\.(on)?)?microsoft\.com['""]?( \|\| \w+)?\s*,[\s\u200b]['""]?)" // pre-match
-                + @"(?<ArgumentPassword>[^`'""\s,;\(\)]+?)" // match
-                + @"(?=[`'""\s,;\(\)])", // post-match
-
-                // AzureActiveDirectoryLoginCredentials
-                  @"" // pre-match
-                + @"(?<PrevPassword>password[\W_][ -~\s\u200b]{40,100}?)" // match
-                + @"(?=@([a-z0-9\.]+\.(on)?)?microsoft\.com)", // post-match
-
-                // LoginCredentials
-                  @"" // pre-match
-                + @"[^a-z\$](DB_USER|user id|uid|(sql)?user(name)?|service\s?account)\s*[^\w\s,]([ -~\s\u200b]{2,120}?|[ -~]{2,30}?)([^a-z\s\$]|\s)\s*(DB_PASS|(sql|service)?password|pwd)\s*[^a-z,\+&\)\]\}\[\{_][ -~\s\u200b]{2,700}?([;|<,})]|$)|[^a-z\s\$]\s*(DB_PASS|password|pwd)\s*[^a-z,\+&\)\]\}\[\{_][ -~\s\u200b]{2,60}?[^a-z\$](DB_USER|user id|uid|user(name)?)\s*[^\w\s,]([ -~\s\u200b]{2,60}?|[ -~]{2,30}?)([;|<,})]|$)" // match
-                + @"", // post-match
-
-                // LoginCredentialsInUrl
-                  @"(?<=(amqp|ssh|(ht|f)tps?)://[^%:\s""'/][^:\s""'/\$]+[^:\s""'/\$%]:)" // pre-match
-                + @"(?<Password>[^%\s""'/][^@\s""'/]{0,100}[^%\s""'/])" // match
-                + @"(?=@[\$a-z0-9:\.\-_%\?=/]+)", // post-match
-
-                // CertificatePrivateKeyHeader
-                  @"" // pre-match
-                + @"(?-i)\-{5}BEGIN( ([DR]SA|EC|OPENSSH|PGP))? PRIVATE KEY( BLOCK)?\-{5}" // match
-                + @"", // post-match
-
-                // HttpAuthorizationHeader
-                  @"(?<=authorization[,\[:= ""']+(basic|digest|hoba|mutual|negotiate|oauth( oauth_token=)?|bearer [^e""'&]|scram\-sha\-1|scram\-sha\-256|vapid|aws4\-hmac\-sha256)[\s\r\n]{0,10})" // pre-match
-                + @"(?<Token>[a-z0-9/+_.=]{10,})" // match
-                + @"", // post-match
-
-                // ClientSecretContext
-                  @"(?<=(^|[a-z\s""'_])((app(lication)?|client)[_\- ]?(key(url)?|secret)|refresh[_\-]?token|[^t]AccessToken(Secret)?|(Consumer|api)[_\- ]?(Secret|Key)|(Twilio(Account|Auth)[_\- ]?(Sid|Token)))([\s=:>]{1,10}|[\s""':=|>,]{3,15}|[""'=:\(]{2}))" // pre-match
-                + @"(?<ClientSecret>(""data:text/plain,.+""|[a-z0-9/+=_.-]{10,200}[^\(\[\{;,\r\n]|[^\s""';<,\)]{5,200}))" // match
-                + @"", // post-match
-
-                // CommunityStringContext
-                  @"(?<=(^|\W{2}|set )snmp(\-server)?( | [ -~]+? )(community|priv)\s[""']?)" // pre-match
-                + @"(?<CommunityString>[^\s]+)" // match
-                + @"(?=[""']?(\s|$))", // post-match
-
-                // PasswordContextInScript
-                  @"(?<=\s-(admin|user|vm)?password\s+[""']?)" // pre-match
-                + @"(?<ScriptArgumentPassword>[^$\(\[<\{\-\s,""']+)[""']?(\s|$)" // match
-                + @"", // post-match
-
-                // PasswordContextInScript
-                  @"(?<=certutil(\.exe)?.{1,10}\-p\s+[""']?)" // pre-match
-                + @"(?<CertUtilPassword>[^\s,]{2,50})" // match
-                + @"(?=[""']?)", // post-match
-
-                // PasswordContextInScript
-                  @"(?<=(^|[_\s\$])[a-z]*(password|secret(key)?)[ \t]*[=:]+[ \t]*)" // pre-match
-                + @"(?<ScriptAssignmentPassword>[^:\s""';,<]{2,200})" // match
-                + @"", // post-match
-
-                // PasswordContextInScript
-                  @"(?<=\s-Name\s+[""']\w+Password[""']\s+-Value\s+[""']?)" // pre-match
-                + @"(?<RegistryPassword>[^\s""']{2,1100})" // match
-                + @"(?=[""']?)", // post-match
-
-                // PasswordContextInScript
-                  @"(?<=(^|[\s\r\n\\])net(\.exe)?[""'\s\\]{1,5}(user\s+|share\s+/user:)[^\s,/]+[ \t]+[""']?)" // pre-match
-                + @"(?<NetUsePassword>[^\s,""'>/]{2,50})" // match
-                + @"(?=[""']?)", // post-match
-
-                // PasswordContextInScript
-                  @"(?<=psexec(\.exe)?.{1,50}-u.{1,50}-p\s+[""']?)" // pre-match
-                + @"(?<PsExecPassword>[^\s,]{2,50})" // match
-                + @"(?=[""']?)", // post-match
-
-                // SymmetricKeyContextInXml
-                  @"" // pre-match
-                + @"<(machineKey|parameter name=""|[a-z]+AccountInfo[^a-z])" // match
-                + @"", // post-match
-
+
+                  // NuGet API keys.
+                  @"\b" // pre-match
+                + @"oy2[a-p][0-9a-z]{15}[aq][0-9a-z]{11}[eu][bdfhjlnprtvxz357][a-p][0-9a-z]{11}[aeimquy4]" // match
+                + @"\b", // post-match
+
+                  // NPM author keys.
+                  @"\b" // pre-match
+                + @"npm_[0-9A-Za-z]{36}" // match
+                + @"\b", // post-match
+
+                  // NPM author keys.
+                  @"\b" // pre-match
+                + @"npm_[0-9A-Za-z]{36}" // match
+                + @"\b", // post-match
             };
     }
 }
diff --git a/src/Test/L0/HostContextL0.cs b/src/Test/L0/HostContextL0.cs
@@ -89,13 +89,27 @@ public void UrlSecretsAreMasked(string input, string expected)
         [Theory]
         [Trait("Level", "L0")]
         [Trait("Category", "Common")]
-        // some secrets that CredScan should suppress
-        [InlineData("xoxr-1xwlcyhsnfn9k69m4efzj3zkfhk", "***")] // Slack token
-        [InlineData("(+n97tcqhcpvu9zkhwwiwx4==)", "(***)")] // 128-bit symmetric key
-        [InlineData("<jwt>eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ.SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c</jwt>", "<jwt>***</jwt>")]
-        // some secrets that CredScan should NOT suppress
-        [InlineData("The password is knock knock knock", "The password is knock knock knock")]
+        // Some secrets that the scanner SHOULD suppress.
+        [InlineData("deaddeaddeaddeaddeaddeaddeaddeadde/dead+deaddeaddeaddeaddeaddeaddeaddeaddeadAPIMxxxxxQ==", "***")]
+        [InlineData("deaddeaddeaddeaddeaddeaddeaddeadde/dead+deaddeaddeaddeaddeaddeaddeaddeaddeadACDbxxxxxQ==", "***")]
+        [InlineData("deaddeaddeaddeaddeaddeaddeaddeadde/dead+deaddeaddeaddeaddeaddeaddeaddeaddead+ABaxxxxxQ==", "***")]
+        [InlineData("deaddeaddeaddeaddeaddeaddeaddeadde/dead+deaddeaddeaddeaddeaddeaddeaddeaddead+AMCxxxxxQ==", "***")]
+        [InlineData("deaddeaddeaddeaddeaddeaddeaddeadde/dead+deaddeaddeaddeaddeaddeaddeaddeaddead+AStxxxxxQ==", "***")]
+        [InlineData("deaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeadAzFuxdeadQ==", "***")]
+        [InlineData("deaddeaddeaddeaddeaddeaddeaddeaddeaddeadxxAzSeDeadxx", "***")]
+        [InlineData("deaddeaddeaddeaddeaddeaddeaddeaddeaddeadde+ACRDeadxx", "***")]
+        [InlineData("oy2mdeaddeaddeadeadqdeaddeadxxxezodeaddeadwxuq", "***")]
+        [InlineData("deaddeaddeaddeaddeaddeaddeaddeadxAIoTDeadxx=", "***")]
+        [InlineData("deaddeaddeaddeaddeaddeaddeaddeadx+ASbDeadxx=", "***")]
+        [InlineData("deaddeaddeaddeaddeaddeaddeaddeadx+AEhDeadxx=", "***")]
+        [InlineData("deaddeaddeaddeaddeaddeaddeaddeadx+ARmDeadxx=", "***")]
+        [InlineData("deaddeaddeaddeaddeaddeaddeaddeaddAzCaDeadxx=", "***")]
+        [InlineData("xxx8Q~dead.dead.DEAD-DEAD-dead~deadxxxxx", "***")]
+        [InlineData("npm_deaddeaddeaddeaddeaddeaddeaddeaddead", "***")]
+        [InlineData("xxx7Q~dead.dead.DEAD-DEAD-dead~deadxx", "***")]
+        // Some secrets that the scanner should NOT suppress.
         [InlineData("SSdtIGEgY29tcGxldGVseSBpbm5vY3VvdXMgc3RyaW5nLg==", "SSdtIGEgY29tcGxldGVseSBpbm5vY3VvdXMgc3RyaW5nLg==")]
+        [InlineData("The password is knock knock knock", "The password is knock knock knock")]
         public void OtherSecretsAreMasked(string input, string expected)
         {
             // Arrange.