Restore expanded redaction as a default feature. Add a set of new, highly accurate detections.

src/Agent.Listener/Configuration/FeatureFlagProvider.cs

@@ -49,7 +49,7 @@ public async Task<FeatureFlag> GetFeatureFlagAsync(IHostContext context, string
             var client = vssConnection.GetClient<FeatureAvailabilityHttpClient>();
             try
             {
-                return await client.GetFeatureFlagByNameAsync(featureFlagName);
+                return await client.GetFeatureFlagByNameAsync(featureFlagName, checkFeatureExists: false);
             } catch(VssServiceException e)
             {
                 Trace.Warning("Unable to retrive feature flag status: " + e.ToString());

src/Agent.Listener/JobDispatcher.cs

@@ -17,7 +17,7 @@
 using System.Linq;
 using Microsoft.VisualStudio.Services.Common;
 using System.Diagnostics;
-
+using Agent.Listener.Configuration;
 
 namespace Microsoft.VisualStudio.Services.Agent.Listener
 {
@@ -88,6 +88,18 @@ public void Run(Pipelines.AgentJobRequestMessage jobRequestMessage, bool runOnce
                 }
             }
 
+            var service = HostContext.GetService<IFeatureFlagProvider>();
+            string ffState;
+            try
+            {
+                ffState = service.GetFeatureFlagAsync(HostContext, "DistributedTask.Agent.EnableAdditionalMaskingRegexes", Trace)?.Result?.EffectiveState;
+            }
+            catch (Exception)
+            {
+                ffState = "Off";
+            }
+            jobRequestMessage.Variables[Constants.Variables.Features.EnableAdditionalMaskingRegexes] = ffState;
+
             WorkerDispatcher newDispatch = new WorkerDispatcher(jobRequestMessage.JobId, jobRequestMessage.RequestId);
             if (runOnce)
             {

src/Agent.Sdk/Knob/AgentKnobs.cs

@@ -284,12 +284,6 @@ public class AgentKnobs
             new EnvironmentKnobSource("SYSTEM_UNSAFEALLOWMULTILINESECRET"),
             new BuiltInDefaultKnobSource("false"));
 
-        public static readonly Knob MaskUsingCredScanRegexes = new Knob(
-            nameof(MaskUsingCredScanRegexes),
-            "Use the CredScan regexes for masking secrets. CredScan is an internal tool developed at Microsoft to keep passwords and authentication keys from being checked in. This defaults to disabled, as there are performance problems with some task outputs.",
-            new EnvironmentKnobSource("AZP_USE_CREDSCAN_REGEXES"),
-            new BuiltInDefaultKnobSource("false"));
-
         public static readonly Knob MaskedSecretMinLength = new Knob(
             nameof(MaskedSecretMinLength),
             "Specify the length of the secrets, which, if shorter, will be ignored in the logs.",

src/Agent.Worker/Worker.cs

@@ -10,6 +10,7 @@
 using System.Threading.Tasks;
 using Microsoft.VisualStudio.Services.WebApi;
 using Agent.Sdk.Util;
+using Agent.Sdk.Knob;
 
 namespace Microsoft.VisualStudio.Services.Agent.Worker
 {
@@ -67,6 +68,22 @@ public async Task<int> RunAsync(string pipeIn, string pipeOut)
                 InitializeSecretMasker(jobMessage);
                 SetCulture(jobMessage);
 
+                var maskUsingCredScanRegexesState = "Off";
+
+                if(jobMessage.Variables.TryGetValue(Constants.Variables.Agent.EnableAdditionalMaskingRegexes, out var enableAdditionalMaskingRegexes))
+                {
+                    maskUsingCredScanRegexesState = enableAdditionalMaskingRegexes.Value;
+                }
+
+                if(maskUsingCredScanRegexesState == "On")
+                {
+                    Trace.Verbose($"{Constants.Variables.Agent.EnableAdditionalMaskingRegexes} is On, adding additional masking regexes");
+                    foreach (var pattern in AdditionalMaskingRegexes.CredScanPatterns)
+                    {
+                        HostContext.SecretMasker.AddRegex(pattern, $"HostContext_{WellKnownSecretAliases.CredScanPatterns}");
+                    }
+                }
+
                 // Start the job.
                 Trace.Info($"Job message:{Environment.NewLine} {StringUtil.ConvertToJson(WorkerUtilities.ScrubPiiData(jobMessage))}");
                 Task<TaskResult> jobRunnerTask = jobRunner.RunAsync(jobMessage, jobRequestCancellationToken.Token);

src/Microsoft.VisualStudio.Services.Agent/AdditionalMaskingRegexes.CredScan.cs

@@ -30,176 +30,55 @@ public static partial class AdditionalMaskingRegexes
         private static IEnumerable<string> credScanPatterns =
             new List<string>()
             {
-                // AnsibleVaultData
-                  @"" // pre-match
-                + @"\$ANSIBLE_VAULT;\d+\.\d+;AES256\s+\d+" // match
+                // AAD client app, most recent two versions.
+                  @"\b" // pre-match
+                + @"[0-9A-Za-z-_~.]{3}7Q~[0-9A-Za-z-_~.]{31}\b|\b[0-9A-Za-z-_~.]{3}8Q~[0-9A-Za-z-_~.]{34}" // match
+                + @"\b", // post-match
+
+                  // Prominent Azure provider 512-bit symmetric keys.
+                  @"\b" // pre-match
+                + @"[0-9A-Za-z+/]{76}(APIM|ACDb|\+(ABa|AMC|ASt))[0-9A-Za-z+/]{5}[AQgw]==" // match
                 + @"", // post-match
-
-                // AzurePowerShellTokenCache
-                  @"" // pre-match
-                + @"[""']TokenCache[""']\s*:\s*\{\s*[""']CacheData[""']\s*:\s*[""'][a-z0-9/\+]{86}" // match
-                + @"", // post-match
-
-                // Base64EncodedStringLiteral
-                  @"(?<=(^|[""'>;=\s#]))" // pre-match
-                + @"(?<DataBlock>(?-i)MI(?i)[a-z0-9/+\s\u0085\u200b""',\\]{200,20000}[a-z0-9/+]={0,2})" // match
+                       // 
+                  // Prominent Azure provider 256-bit symmetric keys.
+                  @"\b" // pre-match
+                + @"[0-9A-Za-z+/]{33}(AIoT|\+(ASb|AEh|ARm))[A-P][0-9A-Za-z+/]{5}=" // match
                 + @"", // post-match
-
-                // JsonWebToken
-                  @"" // pre-match
-                + @"(?-i)(?<JwtToken>eyJ(?i)[a-z0-9\-_%]+\.(?-i)eyJ(?i)[a-z0-9\-_%]+\.[a-z0-9\-_%]+)|([rR]efresh_?[tT]oken|REFRESH_?TOKEN)[""']?\s*[:=]{1,2}\s*[""']?(?<JwtToken>(\w+-)+\w+)[""']?" // match
-                + @"", // post-match
-
-                // SlackTokens
-                  @"" // pre-match
-                + @"xox[pbar]\-[a-z0-9\-]+" // match
-                + @"", // post-match
-
-                // SymmetricKey128
-                  @"(?<=[^\w/\+\._\$,\\])" // pre-match
-                + @"(?<SymmetricKey>[a-z0-9/\+]{22}==)" // match
-                + @"(?=([^\w/\+\.\$]|$))", // post-match
-
-                // SymmetricKey128Hex
-                  @"(?<=[^\w/\+\._\$,\\][dapi]+)" // pre-match
-                + @"(?<SymmetricKey>[a-f0-9]{32})" // match
-                + @"(?=([^\w/\+\.\$]|$))", // post-match
-
-                // SymmetricKey160Hex
-                  @"(?<=[^\w/\+\._\$,\\])" // pre-match
-                + @"(?<Hex160>[a-f0-9/\+]{40})" // match
-                + @"(?=([^\w/\+\.\$]|$))", // post-match
-
-                // SymmetricKey232
-                  @"(?<=[^\w/\+\._\$,\\])" // pre-match
-                + @"(?<SymmetricKey>(?-i)AIza(?i)[a-z0-9_\\\-]{35})" // match
-                + @"(?=([^\w/\+\.\$]|$))", // post-match
-
-                // SymmetricKey240
-                  @"(?<=[^\w/\+\.\-\$,\\])" // pre-match
-                + @"(?<SymmetricKey>[a-z0-9/\+]{40})" // match
-                + @"(?=([^\w/\+\.\-\$,\\]|$))", // post-match
-
-                // SymmetricKey256
-                  @"(?<=[^\w/\+\.\$,\\])" // pre-match
-                + @"(?<SymmetricKey>[a-z0-9/\+]{43}=)" // match
-                + @"(?=([^\w/\+\.\$]|$))", // post-match
-
-                // SymmetricKey256B32
-                  @"(?<=[^\w/\+\._\-\$,\\])" // pre-match
-                + @"(?<SymmetricKey>(?-i)[a-z2-7]{52}(?i))" // match
-                + @"(?=(?<=[0-9]+[a-z]+[0-9]+.{0,49})([^\w/\+\.\-\$,]|$))", // post-match
-
-                // SymmetricKey256UrlEncoded
-                  @"(?<=[^\w/\+\._\-\$,\\%])" // pre-match
-                + @"(?<SymmetricKey>[a-z0-9%]{43,63}%3d)" // match
+
+                  // Azure Function key.
+                  @"\b" // pre-match
+                + @"[0-9A-Za-z_\-]{44}AzFu[0-9A-Za-z\-_]{5}[AQgw]==" // match
                 + @"", // post-match
 
-                // SymmetricKey320
-                  @"(?<=[^\w/\+\.\-\$,\\])" // pre-match
-                + @"(?<SymmetricKey>[a-z0-9/\+]{54}={2})" // match
-                + @"(?=([^\w/\+\.\-\$,\\]|$))", // post-match
-
-                // SymmetricKey320UrlEncoded
-                  @"(?<=[^\w/\+\.\-\$,\\%])" // pre-match
-                + @"(?<SymmetricKey>[a-z0-9%]{54,74}(%3d){2})" // match
+                  // Azure Search keys.
+                  @"\b" // pre-match
+                + @"[0-9A-Za-z]{42}AzSe[A-D][0-9A-Za-z]{5}" // match
+                + @"\b", // post-match
+
+                  // Azure Container Registry keys.
+                  @"\b" // pre-match
+                + @"[0-9A-Za-z+/]{42}\+ACR[A-D][0-9A-Za-z+/]{5}" // match
+                + @"\b", // post-match
+
+                  // Azure Cache for Redis keys.
+                  @"\b" // pre-match
+                + @"[0-9A-Za-z]{33}AzCa[A-P][0-9A-Za-z]{5}=" // match
                 + @"", // post-match
-
-                // SymmetricKey360
-                  @"(?<=[^\w/\+\.\-\$,\\])" // pre-match
-                + @"(?<SymmetricKey>[a-z0-9/\+]{60})" // match
-                + @"(?=[^\w/\+\.\-\$,\\])", // post-match
-
-                // SymmetricKey512
-                  @"(?<=[^\r\n\t\w/\+\.\-\$,\\])" // pre-match
-                + @"(?<SymmetricKey>[a-z0-9/\+]{86}==)" // match
-                + @"(?=([^\w/\+\.\-\$]|$))", // post-match
-
-                // AzureActiveDirectoryLoginCredentials
-                  @"(?<=@([a-z0-9.]+\.(on)?)?microsoft\.com[ -~\s\u200b]{1,80}?(userpass|password|pwd|pw|\wpass[ =:>]+|(get|make)securestring)\W)" // pre-match
-                + @"(?<Password>[^\s;`,""'\(\)]{10,80})" // match
-                + @"(?=[\s;`,""'\(\)])", // post-match
-
-                // AzureActiveDirectoryLoginCredentials
-                  @"" // pre-match
-                + @"(?<MigrationPassword>(\-destinationPasswordPlain ""[^""]+?""))" // match
-                + @"(?=[ -~\s\u200b]{1,150}?@([a-z0-9.]+\.(on)?)?microsoft\.com)", // post-match
-
-                // AzureActiveDirectoryLoginCredentials
-                  @"(?<=(sign_in|SharePointOnlineAuthenticatedContext|(User|Exchange)Credentials?|password)[ -~\s\u200b]{1,100}?@([a-z0-9.]+\.(on)?)?microsoft\.com['""]?( \|\| \w+)?\s*,[\s\u200b]['""]?)" // pre-match
-                + @"(?<ArgumentPassword>[^`'""\s,;\(\)]+?)" // match
-                + @"(?=[`'""\s,;\(\)])", // post-match
-
-                // AzureActiveDirectoryLoginCredentials
-                  @"" // pre-match
-                + @"(?<PrevPassword>password[\W_][ -~\s\u200b]{40,100}?)" // match
-                + @"(?=@([a-z0-9\.]+\.(on)?)?microsoft\.com)", // post-match
-
-                // LoginCredentials
-                  @"" // pre-match
-                + @"[^a-z\$](DB_USER|user id|uid|(sql)?user(name)?|service\s?account)\s*[^\w\s,]([ -~\s\u200b]{2,120}?|[ -~]{2,30}?)([^a-z\s\$]|\s)\s*(DB_PASS|(sql|service)?password|pwd)\s*[^a-z,\+&\)\]\}\[\{_][ -~\s\u200b]{2,700}?([;|<,})]|$)|[^a-z\s\$]\s*(DB_PASS|password|pwd)\s*[^a-z,\+&\)\]\}\[\{_][ -~\s\u200b]{2,60}?[^a-z\$](DB_USER|user id|uid|user(name)?)\s*[^\w\s,]([ -~\s\u200b]{2,60}?|[ -~]{2,30}?)([;|<,})]|$)" // match
-                + @"", // post-match
-
-                // LoginCredentialsInUrl
-                  @"(?<=(amqp|ssh|(ht|f)tps?)://[^%:\s""'/][^:\s""'/\$]+[^:\s""'/\$%]:)" // pre-match
-                + @"(?<Password>[^%\s""'/][^@\s""'/]{0,100}[^%\s""'/])" // match
-                + @"(?=@[\$a-z0-9:\.\-_%\?=/]+)", // post-match
-
-                // CertificatePrivateKeyHeader
-                  @"" // pre-match
-                + @"(?-i)\-{5}BEGIN( ([DR]SA|EC|OPENSSH|PGP))? PRIVATE KEY( BLOCK)?\-{5}" // match
-                + @"", // post-match
-
-                // HttpAuthorizationHeader
-                  @"(?<=authorization[,\[:= ""']+(basic|digest|hoba|mutual|negotiate|oauth( oauth_token=)?|bearer [^e""'&]|scram\-sha\-1|scram\-sha\-256|vapid|aws4\-hmac\-sha256)[\s\r\n]{0,10})" // pre-match
-                + @"(?<Token>[a-z0-9/+_.=]{10,})" // match
-                + @"", // post-match
-
-                // ClientSecretContext
-                  @"(?<=(^|[a-z\s""'_])((app(lication)?|client)[_\- ]?(key(url)?|secret)|refresh[_\-]?token|[^t]AccessToken(Secret)?|(Consumer|api)[_\- ]?(Secret|Key)|(Twilio(Account|Auth)[_\- ]?(Sid|Token)))([\s=:>]{1,10}|[\s""':=|>,]{3,15}|[""'=:\(]{2}))" // pre-match
-                + @"(?<ClientSecret>(""data:text/plain,.+""|[a-z0-9/+=_.-]{10,200}[^\(\[\{;,\r\n]|[^\s""';<,\)]{5,200}))" // match
-                + @"", // post-match
-
-                // CommunityStringContext
-                  @"(?<=(^|\W{2}|set )snmp(\-server)?( | [ -~]+? )(community|priv)\s[""']?)" // pre-match
-                + @"(?<CommunityString>[^\s]+)" // match
-                + @"(?=[""']?(\s|$))", // post-match
-
-                // PasswordContextInScript
-                  @"(?<=\s-(admin|user|vm)?password\s+[""']?)" // pre-match
-                + @"(?<ScriptArgumentPassword>[^$\(\[<\{\-\s,""']+)[""']?(\s|$)" // match
-                + @"", // post-match
-
-                // PasswordContextInScript
-                  @"(?<=certutil(\.exe)?.{1,10}\-p\s+[""']?)" // pre-match
-                + @"(?<CertUtilPassword>[^\s,]{2,50})" // match
-                + @"(?=[""']?)", // post-match
-
-                // PasswordContextInScript
-                  @"(?<=(^|[_\s\$])[a-z]*(password|secret(key)?)[ \t]*[=:]+[ \t]*)" // pre-match
-                + @"(?<ScriptAssignmentPassword>[^:\s""';,<]{2,200})" // match
-                + @"", // post-match
-
-                // PasswordContextInScript
-                  @"(?<=\s-Name\s+[""']\w+Password[""']\s+-Value\s+[""']?)" // pre-match
-                + @"(?<RegistryPassword>[^\s""']{2,1100})" // match
-                + @"(?=[""']?)", // post-match
-
-                // PasswordContextInScript
-                  @"(?<=(^|[\s\r\n\\])net(\.exe)?[""'\s\\]{1,5}(user\s+|share\s+/user:)[^\s,/]+[ \t]+[""']?)" // pre-match
-                + @"(?<NetUsePassword>[^\s,""'>/]{2,50})" // match
-                + @"(?=[""']?)", // post-match
-
-                // PasswordContextInScript
-                  @"(?<=psexec(\.exe)?.{1,50}-u.{1,50}-p\s+[""']?)" // pre-match
-                + @"(?<PsExecPassword>[^\s,]{2,50})" // match
-                + @"(?=[""']?)", // post-match
-
-                // SymmetricKeyContextInXml
-                  @"" // pre-match
-                + @"<(machineKey|parameter name=""|[a-z]+AccountInfo[^a-z])" // match
-                + @"", // post-match
-
+
+                  // NuGet API keys.
+                  @"\b" // pre-match
+                + @"oy2[a-p][0-9a-z]{15}[aq][0-9a-z]{11}[eu][bdfhjlnprtvxz357][a-p][0-9a-z]{11}[aeimquy4]" // match
+                + @"\b", // post-match
+
+                  // NPM author keys.
+                  @"\b" // pre-match
+                + @"npm_[0-9A-Za-z]{36}" // match
+                + @"\b", // post-match
+
+                  // NPM author keys.
+                  @"\b" // pre-match
+                + @"npm_[0-9A-Za-z]{36}" // match
+                + @"\b", // post-match
             };
     }
 }

src/Microsoft.VisualStudio.Services.Agent/Constants.cs

@@ -322,6 +322,7 @@ public static class Agent
                 public static readonly string Version = "agent.version";
                 public static readonly string WorkFolder = "agent.workfolder";
                 public static readonly string WorkingDirectory = "agent.WorkingDirectory";
+                public static readonly string EnableAdditionalMaskingRegexes = "agent.enableadditionalmaskingregexes";
             }
 
             public static class Build
@@ -371,6 +372,7 @@ public static class Features
                 public static readonly string GitLfsSupport = "agent.source.git.lfs";
                 public static readonly string GitShallowDepth = "agent.source.git.shallowFetchDepth";
                 public static readonly string SkipSyncSource = "agent.source.skip";
+                public static readonly string EnableAdditionalMaskingRegexes = "agent.enableadditionalmaskingregexes";
             }
 
             public static class Maintenance

src/Microsoft.VisualStudio.Services.Agent/HostContext.cs

@@ -21,6 +21,7 @@
 using System.Net.Http.Headers;
 using Pipelines = Microsoft.TeamFoundation.DistributedTask.Pipelines;
 using Agent.Sdk.Util;
+using BuildXL.Cache.ContentStore.Interfaces.Tracing;
 
 namespace Microsoft.VisualStudio.Services.Agent
 {
@@ -88,6 +89,7 @@ public class HostContext : EventListener, IObserver<DiagnosticListener>, IObserv
         public ShutdownReason AgentShutdownReason { get; private set; }
         public ILoggedSecretMasker SecretMasker => _secretMasker;
         public ProductInfoHeaderValue UserAgent => _userAgent;
+
         public HostContext(HostType hostType, string logFile = null)
         {
             _secretMasker = new LoggedSecretMasker(_basicSecretMasker);
@@ -106,13 +108,6 @@ public HostContext(HostType hostType, string logFile = null)
             this.SecretMasker.AddValueEncoder(ValueEncoders.UriDataEscape, $"HostContext_{WellKnownSecretAliases.UriDataEscape}");
             this.SecretMasker.AddValueEncoder(ValueEncoders.BackslashEscape, $"HostContext_{WellKnownSecretAliases.UriDataEscape}");
             this.SecretMasker.AddRegex(AdditionalMaskingRegexes.UrlSecretPattern, $"HostContext_{WellKnownSecretAliases.UrlSecretPattern}");
-            if (AgentKnobs.MaskUsingCredScanRegexes.GetValue(this).AsBoolean())
-            {
-                foreach (var pattern in AdditionalMaskingRegexes.CredScanPatterns)
-                {
-                    this.SecretMasker.AddRegex(pattern, $"HostContext_{WellKnownSecretAliases.CredScanPatterns}");
-                }
-            }
 
             // Create the trace manager.
             if (string.IsNullOrEmpty(logFile))