From 6e7b55a5c06f5b15da9664e39f8680e1b557f207 Mon Sep 17 00:00:00 2001 From: "Michael C. Fanning" Date: Wed, 11 Oct 2023 21:04:59 -0700 Subject: [PATCH] Restore expanded redaction as a default feature. Add a set of new, highly accurate detections. --- src/Agent.Sdk/Knob/AgentKnobs.cs | 2 +- .../AdditionalMaskingRegexes.CredScan.cs | 209 ++++-------------- src/Test/L0/HostContextL0.cs | 26 ++- 3 files changed, 65 insertions(+), 172 deletions(-) diff --git a/src/Agent.Sdk/Knob/AgentKnobs.cs b/src/Agent.Sdk/Knob/AgentKnobs.cs index cbe5fdab95..6a5d09186a 100644 --- a/src/Agent.Sdk/Knob/AgentKnobs.cs +++ b/src/Agent.Sdk/Knob/AgentKnobs.cs @@ -288,7 +288,7 @@ public class AgentKnobs nameof(MaskUsingCredScanRegexes), "Use the CredScan regexes for masking secrets. CredScan is an internal tool developed at Microsoft to keep passwords and authentication keys from being checked in. This defaults to disabled, as there are performance problems with some task outputs.", new EnvironmentKnobSource("AZP_USE_CREDSCAN_REGEXES"), - new BuiltInDefaultKnobSource("false")); + new BuiltInDefaultKnobSource("true")); public static readonly Knob MaskedSecretMinLength = new Knob( nameof(MaskedSecretMinLength), diff --git a/src/Microsoft.VisualStudio.Services.Agent/AdditionalMaskingRegexes.CredScan.cs b/src/Microsoft.VisualStudio.Services.Agent/AdditionalMaskingRegexes.CredScan.cs index a2af9a961f..48273c5a0e 100644 --- a/src/Microsoft.VisualStudio.Services.Agent/AdditionalMaskingRegexes.CredScan.cs +++ b/src/Microsoft.VisualStudio.Services.Agent/AdditionalMaskingRegexes.CredScan.cs @@ -30,176 +30,55 @@ public static partial class AdditionalMaskingRegexes private static IEnumerable credScanPatterns = new List() { - // AnsibleVaultData - @"" // pre-match - + @"\$ANSIBLE_VAULT;\d+\.\d+;AES256\s+\d+" // match + // AAD client app, most recent two versions. + @"\b" // pre-match + + @"[0-9A-Za-z-_~.]{3}7Q~[0-9A-Za-z-_~.]{31}\b|\b[0-9A-Za-z-_~.]{3}8Q~[0-9A-Za-z-_~.]{34}" // match + + @"\b", // post-match + + // Prominent Azure provider 512-bit symmetric keys. + @"\b" // pre-match + + @"[0-9A-Za-z+/]{76}(APIM|ACDb|\+(ABa|AMC|ASt))[0-9A-Za-z+/]{5}[AQgw]==" // match + @"", // post-match - - // AzurePowerShellTokenCache - @"" // pre-match - + @"[""']TokenCache[""']\s*:\s*\{\s*[""']CacheData[""']\s*:\s*[""'][a-z0-9/\+]{86}" // match - + @"", // post-match - - // Base64EncodedStringLiteral - @"(?<=(^|[""'>;=\s#]))" // pre-match - + @"(?(?-i)MI(?i)[a-z0-9/+\s\u0085\u200b""',\\]{200,20000}[a-z0-9/+]={0,2})" // match + // + // Prominent Azure provider 256-bit symmetric keys. + @"\b" // pre-match + + @"[0-9A-Za-z+/]{33}(AIoT|\+(ASb|AEh|ARm))[A-P][0-9A-Za-z+/]{5}=" // match + @"", // post-match - - // JsonWebToken - @"" // pre-match - + @"(?-i)(?eyJ(?i)[a-z0-9\-_%]+\.(?-i)eyJ(?i)[a-z0-9\-_%]+\.[a-z0-9\-_%]+)|([rR]efresh_?[tT]oken|REFRESH_?TOKEN)[""']?\s*[:=]{1,2}\s*[""']?(?(\w+-)+\w+)[""']?" // match - + @"", // post-match - - // SlackTokens - @"" // pre-match - + @"xox[pbar]\-[a-z0-9\-]+" // match - + @"", // post-match - - // SymmetricKey128 - @"(?<=[^\w/\+\._\$,\\])" // pre-match - + @"(?[a-z0-9/\+]{22}==)" // match - + @"(?=([^\w/\+\.\$]|$))", // post-match - - // SymmetricKey128Hex - @"(?<=[^\w/\+\._\$,\\][dapi]+)" // pre-match - + @"(?[a-f0-9]{32})" // match - + @"(?=([^\w/\+\.\$]|$))", // post-match - - // SymmetricKey160Hex - @"(?<=[^\w/\+\._\$,\\])" // pre-match - + @"(?[a-f0-9/\+]{40})" // match - + @"(?=([^\w/\+\.\$]|$))", // post-match - - // SymmetricKey232 - @"(?<=[^\w/\+\._\$,\\])" // pre-match - + @"(?(?-i)AIza(?i)[a-z0-9_\\\-]{35})" // match - + @"(?=([^\w/\+\.\$]|$))", // post-match - - // SymmetricKey240 - @"(?<=[^\w/\+\.\-\$,\\])" // pre-match - + @"(?[a-z0-9/\+]{40})" // match - + @"(?=([^\w/\+\.\-\$,\\]|$))", // post-match - - // SymmetricKey256 - @"(?<=[^\w/\+\.\$,\\])" // pre-match - + @"(?[a-z0-9/\+]{43}=)" // match - + @"(?=([^\w/\+\.\$]|$))", // post-match - - // SymmetricKey256B32 - @"(?<=[^\w/\+\._\-\$,\\])" // pre-match - + @"(?(?-i)[a-z2-7]{52}(?i))" // match - + @"(?=(?<=[0-9]+[a-z]+[0-9]+.{0,49})([^\w/\+\.\-\$,]|$))", // post-match - - // SymmetricKey256UrlEncoded - @"(?<=[^\w/\+\._\-\$,\\%])" // pre-match - + @"(?[a-z0-9%]{43,63}%3d)" // match + + // Azure Function key. + @"\b" // pre-match + + @"[0-9A-Za-z_\-]{44}AzFu[0-9A-Za-z\-_]{5}[AQgw]==" // match + @"", // post-match - // SymmetricKey320 - @"(?<=[^\w/\+\.\-\$,\\])" // pre-match - + @"(?[a-z0-9/\+]{54}={2})" // match - + @"(?=([^\w/\+\.\-\$,\\]|$))", // post-match - - // SymmetricKey320UrlEncoded - @"(?<=[^\w/\+\.\-\$,\\%])" // pre-match - + @"(?[a-z0-9%]{54,74}(%3d){2})" // match + // Azure Search query and admin keys. + @"\b" // pre-match + + @"[0-9A-Za-z]{42}AzSe[A-D][0-9A-Za-z]{5}" // match + + @"\b", // post-match + + // Azure Container Registry keys. + @"\b" // pre-match + + @"[0-9A-Za-z+/]{42}\+ACR[A-D][0-9A-Za-z+/]{5}" // match + + @"\b", // post-match + + // Azure Cache for Redis keys. + @"\b" // pre-match + + @"[0-9A-Za-z]{33}AzCa[A-P][0-9A-Za-z]{5}=" // match + @"", // post-match - - // SymmetricKey360 - @"(?<=[^\w/\+\.\-\$,\\])" // pre-match - + @"(?[a-z0-9/\+]{60})" // match - + @"(?=[^\w/\+\.\-\$,\\])", // post-match - - // SymmetricKey512 - @"(?<=[^\r\n\t\w/\+\.\-\$,\\])" // pre-match - + @"(?[a-z0-9/\+]{86}==)" // match - + @"(?=([^\w/\+\.\-\$]|$))", // post-match - - // AzureActiveDirectoryLoginCredentials - @"(?<=@([a-z0-9.]+\.(on)?)?microsoft\.com[ -~\s\u200b]{1,80}?(userpass|password|pwd|pw|\wpass[ =:>]+|(get|make)securestring)\W)" // pre-match - + @"(?[^\s;`,""'\(\)]{10,80})" // match - + @"(?=[\s;`,""'\(\)])", // post-match - - // AzureActiveDirectoryLoginCredentials - @"" // pre-match - + @"(?(\-destinationPasswordPlain ""[^""]+?""))" // match - + @"(?=[ -~\s\u200b]{1,150}?@([a-z0-9.]+\.(on)?)?microsoft\.com)", // post-match - - // AzureActiveDirectoryLoginCredentials - @"(?<=(sign_in|SharePointOnlineAuthenticatedContext|(User|Exchange)Credentials?|password)[ -~\s\u200b]{1,100}?@([a-z0-9.]+\.(on)?)?microsoft\.com['""]?( \|\| \w+)?\s*,[\s\u200b]['""]?)" // pre-match - + @"(?[^`'""\s,;\(\)]+?)" // match - + @"(?=[`'""\s,;\(\)])", // post-match - - // AzureActiveDirectoryLoginCredentials - @"" // pre-match - + @"(?password[\W_][ -~\s\u200b]{40,100}?)" // match - + @"(?=@([a-z0-9\.]+\.(on)?)?microsoft\.com)", // post-match - - // LoginCredentials - @"" // pre-match - + @"[^a-z\$](DB_USER|user id|uid|(sql)?user(name)?|service\s?account)\s*[^\w\s,]([ -~\s\u200b]{2,120}?|[ -~]{2,30}?)([^a-z\s\$]|\s)\s*(DB_PASS|(sql|service)?password|pwd)\s*[^a-z,\+&\)\]\}\[\{_][ -~\s\u200b]{2,700}?([;|<,})]|$)|[^a-z\s\$]\s*(DB_PASS|password|pwd)\s*[^a-z,\+&\)\]\}\[\{_][ -~\s\u200b]{2,60}?[^a-z\$](DB_USER|user id|uid|user(name)?)\s*[^\w\s,]([ -~\s\u200b]{2,60}?|[ -~]{2,30}?)([;|<,})]|$)" // match - + @"", // post-match - - // LoginCredentialsInUrl - @"(?<=(amqp|ssh|(ht|f)tps?)://[^%:\s""'/][^:\s""'/\$]+[^:\s""'/\$%]:)" // pre-match - + @"(?[^%\s""'/][^@\s""'/]{0,100}[^%\s""'/])" // match - + @"(?=@[\$a-z0-9:\.\-_%\?=/]+)", // post-match - - // CertificatePrivateKeyHeader - @"" // pre-match - + @"(?-i)\-{5}BEGIN( ([DR]SA|EC|OPENSSH|PGP))? PRIVATE KEY( BLOCK)?\-{5}" // match - + @"", // post-match - - // HttpAuthorizationHeader - @"(?<=authorization[,\[:= ""']+(basic|digest|hoba|mutual|negotiate|oauth( oauth_token=)?|bearer [^e""'&]|scram\-sha\-1|scram\-sha\-256|vapid|aws4\-hmac\-sha256)[\s\r\n]{0,10})" // pre-match - + @"(?[a-z0-9/+_.=]{10,})" // match - + @"", // post-match - - // ClientSecretContext - @"(?<=(^|[a-z\s""'_])((app(lication)?|client)[_\- ]?(key(url)?|secret)|refresh[_\-]?token|[^t]AccessToken(Secret)?|(Consumer|api)[_\- ]?(Secret|Key)|(Twilio(Account|Auth)[_\- ]?(Sid|Token)))([\s=:>]{1,10}|[\s""':=|>,]{3,15}|[""'=:\(]{2}))" // pre-match - + @"(?(""data:text/plain,.+""|[a-z0-9/+=_.-]{10,200}[^\(\[\{;,\r\n]|[^\s""';<,\)]{5,200}))" // match - + @"", // post-match - - // CommunityStringContext - @"(?<=(^|\W{2}|set )snmp(\-server)?( | [ -~]+? )(community|priv)\s[""']?)" // pre-match - + @"(?[^\s]+)" // match - + @"(?=[""']?(\s|$))", // post-match - - // PasswordContextInScript - @"(?<=\s-(admin|user|vm)?password\s+[""']?)" // pre-match - + @"(?[^$\(\[<\{\-\s,""']+)[""']?(\s|$)" // match - + @"", // post-match - - // PasswordContextInScript - @"(?<=certutil(\.exe)?.{1,10}\-p\s+[""']?)" // pre-match - + @"(?[^\s,]{2,50})" // match - + @"(?=[""']?)", // post-match - - // PasswordContextInScript - @"(?<=(^|[_\s\$])[a-z]*(password|secret(key)?)[ \t]*[=:]+[ \t]*)" // pre-match - + @"(?[^:\s""';,<]{2,200})" // match - + @"", // post-match - - // PasswordContextInScript - @"(?<=\s-Name\s+[""']\w+Password[""']\s+-Value\s+[""']?)" // pre-match - + @"(?[^\s""']{2,1100})" // match - + @"(?=[""']?)", // post-match - - // PasswordContextInScript - @"(?<=(^|[\s\r\n\\])net(\.exe)?[""'\s\\]{1,5}(user\s+|share\s+/user:)[^\s,/]+[ \t]+[""']?)" // pre-match - + @"(?[^\s,""'>/]{2,50})" // match - + @"(?=[""']?)", // post-match - - // PasswordContextInScript - @"(?<=psexec(\.exe)?.{1,50}-u.{1,50}-p\s+[""']?)" // pre-match - + @"(?[^\s,]{2,50})" // match - + @"(?=[""']?)", // post-match - - // SymmetricKeyContextInXml - @"" // pre-match - + @"<(machineKey|parameter name=""|[a-z]+AccountInfo[^a-z])" // match - + @"", // post-match - + + // NuGet API keys. + @"\b" // pre-match + + @"oy2[a-p][0-9a-z]{15}[aq][0-9a-z]{11}[eu][bdfhjlnprtvxz357][a-p][0-9a-z]{11}[aeimquy4]" // match + + @"\b", // post-match + + // NPM author keys. + @"\b" // pre-match + + @"npm_[0-9A-Za-z]{36}" // match + + @"\b", // post-match + + // NPM author keys. + @"\b" // pre-match + + @"npm_[0-9A-Za-z]{36}" // match + + @"\b", // post-match }; } } diff --git a/src/Test/L0/HostContextL0.cs b/src/Test/L0/HostContextL0.cs index ede7406606..a41c215bec 100644 --- a/src/Test/L0/HostContextL0.cs +++ b/src/Test/L0/HostContextL0.cs @@ -89,13 +89,27 @@ public void UrlSecretsAreMasked(string input, string expected) [Theory] [Trait("Level", "L0")] [Trait("Category", "Common")] - // some secrets that CredScan should suppress - [InlineData("xoxr-1xwlcyhsnfn9k69m4efzj3zkfhk", "***")] // Slack token - [InlineData("(+n97tcqhcpvu9zkhwwiwx4==)", "(***)")] // 128-bit symmetric key - [InlineData("eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ.SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c", "***")] - // some secrets that CredScan should NOT suppress - [InlineData("The password is knock knock knock", "The password is knock knock knock")] + // Some secrets that the scanner SHOULD suppress. + [InlineData("deaddeaddeaddeaddeaddeaddeaddeadde/dead+deaddeaddeaddeaddeaddeaddeaddeaddeadAPIMxxxxxQ==", "***")] + [InlineData("deaddeaddeaddeaddeaddeaddeaddeadde/dead+deaddeaddeaddeaddeaddeaddeaddeaddeadACDbxxxxxQ==", "***")] + [InlineData("deaddeaddeaddeaddeaddeaddeaddeadde/dead+deaddeaddeaddeaddeaddeaddeaddeaddead+ABaxxxxxQ==", "***")] + [InlineData("deaddeaddeaddeaddeaddeaddeaddeadde/dead+deaddeaddeaddeaddeaddeaddeaddeaddead+AMCxxxxxQ==", "***")] + [InlineData("deaddeaddeaddeaddeaddeaddeaddeadde/dead+deaddeaddeaddeaddeaddeaddeaddeaddead+AStxxxxxQ==", "***")] + [InlineData("deaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeadAzFuxdeadQ==", "***")] + [InlineData("deaddeaddeaddeaddeaddeaddeaddeaddeaddeadxxAzSeDeadxx", "***")] + [InlineData("deaddeaddeaddeaddeaddeaddeaddeaddeaddeadde+ACRDeadxx", "***")] + [InlineData("oy2mdeaddeaddeadeadqdeaddeadxxxezodeaddeadwxuq", "***")] + [InlineData("deaddeaddeaddeaddeaddeaddeaddeadxAIoTDeadxx=", "***")] + [InlineData("deaddeaddeaddeaddeaddeaddeaddeadx+ASbDeadxx=", "***")] + [InlineData("deaddeaddeaddeaddeaddeaddeaddeadx+AEhDeadxx=", "***")] + [InlineData("deaddeaddeaddeaddeaddeaddeaddeadx+ARmDeadxx=", "***")] + [InlineData("deaddeaddeaddeaddeaddeaddeaddeaddAzCaDeadxx=", "***")] + [InlineData("xxx8Q~dead.dead.DEAD-DEAD-dead~deadxxxxx", "***")] + [InlineData("npm_deaddeaddeaddeaddeaddeaddeaddeaddead", "***")] + [InlineData("xxx7Q~dead.dead.DEAD-DEAD-dead~deadxx", "***")] + // Some secrets that the scanner should NOT suppress. [InlineData("SSdtIGEgY29tcGxldGVseSBpbm5vY3VvdXMgc3RyaW5nLg==", "SSdtIGEgY29tcGxldGVseSBpbm5vY3VvdXMgc3RyaW5nLg==")] + [InlineData("The password is knock knock knock", "The password is knock knock knock")] public void OtherSecretsAreMasked(string input, string expected) { // Arrange.