Merge branch 'master' into users/ismayilov-ismayil/remove_node6_runner

microsoft · Oct 16, 2023 · f575bea · f575bea
2 parents 88bb0e4 + 34e2c30
commit f575bea
Show file tree

Hide file tree

Showing 26 changed files with 954 additions and 518 deletions.
diff --git a/.azure-pipelines/build-job.yml b/.azure-pipelines/build-job.yml
@@ -138,6 +138,7 @@ jobs:
     - script: ${{ variables.devCommand }} testl0 Debug ${{ parameters.os }}-${{ parameters.arch }}
       workingDirectory: src
       displayName: Unit tests
+      timeoutInMinutes: 5
 
   # Install nuget
   - ${{ if eq(parameters.os, 'win') }}:

diff --git a/src/Agent.Listener/Configuration/FeatureFlagProvider.cs b/src/Agent.Listener/Configuration/FeatureFlagProvider.cs
@@ -49,7 +49,7 @@ public async Task<FeatureFlag> GetFeatureFlagAsync(IHostContext context, string
             var client = vssConnection.GetClient<FeatureAvailabilityHttpClient>();
             try
             {
-                return await client.GetFeatureFlagByNameAsync(featureFlagName);
+                return await client.GetFeatureFlagByNameAsync(featureFlagName, checkFeatureExists: false);
             } catch(VssServiceException e)
             {
                 Trace.Warning("Unable to retrive feature flag status: " + e.ToString());

diff --git a/src/Agent.Listener/JobDispatcher.cs b/src/Agent.Listener/JobDispatcher.cs
@@ -17,7 +17,7 @@
 using System.Linq;
 using Microsoft.VisualStudio.Services.Common;
 using System.Diagnostics;
-
+using Agent.Listener.Configuration;
 
 namespace Microsoft.VisualStudio.Services.Agent.Listener
 {
@@ -88,6 +88,18 @@ public void Run(Pipelines.AgentJobRequestMessage jobRequestMessage, bool runOnce
                 }
             }
 
+            var service = HostContext.GetService<IFeatureFlagProvider>();
+            string ffState;
+            try
+            {
+                ffState = service.GetFeatureFlagAsync(HostContext, "DistributedTask.Agent.EnableAdditionalMaskingRegexes", Trace)?.Result?.EffectiveState;
+            }
+            catch (Exception)
+            {
+                ffState = "Off";
+            }
+            jobRequestMessage.Variables[Constants.Variables.Features.EnableAdditionalMaskingRegexes] = ffState;
+
             WorkerDispatcher newDispatch = new WorkerDispatcher(jobRequestMessage.JobId, jobRequestMessage.RequestId);
             if (runOnce)
             {

diff --git a/src/Agent.Listener/MessageListener.cs b/src/Agent.Listener/MessageListener.cs
@@ -261,6 +261,9 @@ public async Task<TaskAgentMessage> GetNextMessageAsync(CancellationToken token)
                         Trace.Verbose($"No message retrieved from session '{_session.SessionId}'.");
                     }
 
+                    _getNextMessageRetryInterval = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(5), TimeSpan.FromSeconds(15), _getNextMessageRetryInterval);
+                    Trace.Info("Sleeping for {0} seconds before retrying.", _getNextMessageRetryInterval.TotalSeconds);
+                    await HostContext.Delay(_getNextMessageRetryInterval, token);
                     continue;
                 }
 

diff --git a/src/Agent.Plugins/Artifact/PipelineArtifactConstants.cs b/src/Agent.Plugins/Artifact/PipelineArtifactConstants.cs
@@ -3,6 +3,7 @@
 
 namespace Agent.Plugins
 {
+    // Use PipelineArtifactContants.cs from ADO, once the latest libs are available.
     public class PipelineArtifactConstants
     {
         public const string AzurePipelinesAgent = "AzurePipelinesAgent";
@@ -18,5 +19,6 @@ public class PipelineArtifactConstants
         public const string FileShareArtifact = "filepath";
         public const string CustomPropertiesPrefix = "user-";
         public const string HashType = "HashType";
+        public const string DomainId = "DomainId";
     }
 }
diff --git a/src/Agent.Plugins/Artifact/PipelineArtifactProvider.cs b/src/Agent.Plugins/Artifact/PipelineArtifactProvider.cs
@@ -15,6 +15,7 @@
 using Microsoft.VisualStudio.Services.WebApi;
 using Microsoft.VisualStudio.Services.Content.Common;
 using Microsoft.VisualStudio.Services.BlobStore.Common;
+using Microsoft.VisualStudio.Services.BlobStore.Common.Telemetry;
 
 namespace Agent.Plugins
 {
@@ -37,12 +38,19 @@ public async Task DownloadSingleArtifactAsync(
             CancellationToken cancellationToken,
             AgentTaskPluginExecutionContext context)
         {
+            // if  properties doesn't have it, use the default domain for backward compatibility
+            IDomainId domainId = WellKnownDomainIds.DefaultDomainId;
+            if(buildArtifact.Resource.Properties.TryGetValue(PipelineArtifactConstants.DomainId, out string domainIdString))
+            {
+                domainId = DomainIdFactory.Create(domainIdString);
+            }
+
             var (dedupManifestClient, clientTelemetry) = await DedupManifestArtifactClientFactory.Instance.CreateDedupManifestClientAsync(
                 this.context.IsSystemDebugTrue(),
                 (str) => this.context.Output(str),
                 this.connection,
                 DedupManifestArtifactClientFactory.Instance.GetDedupStoreClientMaxParallelism(context),
-                WellKnownDomainIds.DefaultDomainId,
+                domainId,
                 Microsoft.VisualStudio.Services.BlobStore.WebApi.Contracts.Client.PipelineArtifact,
                 context,
                 cancellationToken);
@@ -85,49 +93,81 @@ public async Task DownloadMultipleArtifactsAsync(
             CancellationToken cancellationToken,
             AgentTaskPluginExecutionContext context)
         {
-            var (dedupManifestClient, clientTelemetry) = await DedupManifestArtifactClientFactory.Instance.CreateDedupManifestClientAsync(
-                this.context.IsSystemDebugTrue(),
-                (str) => this.context.Output(str),
-                this.connection,
-                DedupManifestArtifactClientFactory.Instance.GetDedupStoreClientMaxParallelism(context),
-                WellKnownDomainIds.DefaultDomainId,
-                Microsoft.VisualStudio.Services.BlobStore.WebApi.Contracts.Client.PipelineArtifact,
-                context,
-                cancellationToken);
+            // create clients and group artifacts for each domain:
+            Dictionary<IDomainId, (DedupManifestArtifactClient Client, BlobStoreClientTelemetry Telemetry, Dictionary<string, DedupIdentifier> ArtifactDictionary)> dedupManifestClients = 
+                new();
 
-            using (clientTelemetry)
-            {
-                var artifactNameAndManifestIds = buildArtifacts.ToDictionary(
-                    keySelector: (a) => a.Name, // keys should be unique, if not something is really wrong
-                    elementSelector: (a) => DedupIdentifier.Create(a.Resource.Data));
-                // 2) download to the target path
-                var options = DownloadDedupManifestArtifactOptions.CreateWithMultiManifestIds(
-                    artifactNameAndManifestIds,
-                    downloadParameters.TargetDirectory,
-                    proxyUri: null,
-                    minimatchPatterns: downloadParameters.MinimatchFilters,
-                    minimatchFilterWithArtifactName: downloadParameters.MinimatchFilterWithArtifactName);
+            foreach(var buildArtifact in buildArtifacts)
+            {                
+                // if  properties doesn't have it, use the default domain for backward compatibility
+                IDomainId domainId = WellKnownDomainIds.DefaultDomainId;
+                if(buildArtifact.Resource.Properties.TryGetValue(PipelineArtifactConstants.DomainId, out string domainIdString))
+                {
+                    domainId = DomainIdFactory.Create(domainIdString);
+                }
 
-                PipelineArtifactActionRecord downloadRecord = clientTelemetry.CreateRecord<PipelineArtifactActionRecord>((level, uri, type) =>
-                    new PipelineArtifactActionRecord(level, uri, type, nameof(DownloadMultipleArtifactsAsync), this.context));
-                await clientTelemetry.MeasureActionAsync(
-                    record: downloadRecord,
-                    actionAsync: async () =>
+                // Have we already created the clients for this domain?
+                if(dedupManifestClients.ContainsKey(domainId)) {
+                    // Clients already created for this domain, Just add the artifact to the list:
+                    dedupManifestClients[domainId].ArtifactDictionary.Add(buildArtifact.Name, DedupIdentifier.Create(buildArtifact.Resource.Data));
+                }
+                else
+                {
+                    // create the clients:
+                    var (dedupManifestClient, clientTelemetry) = await DedupManifestArtifactClientFactory.Instance.CreateDedupManifestClientAsync(
+                        this.context.IsSystemDebugTrue(),
+                        (str) => this.context.Output(str),
+                        this.connection,
+                        DedupManifestArtifactClientFactory.Instance.GetDedupStoreClientMaxParallelism(context),
+                        domainId,
+                        Microsoft.VisualStudio.Services.BlobStore.WebApi.Contracts.Client.PipelineArtifact,
+                        context,
+                        cancellationToken);
+
+                    // and create the artifact dictionary with the current artifact
+                    var artifactDictionary = new Dictionary<string, DedupIdentifier>
                     {
-                        await AsyncHttpRetryHelper.InvokeVoidAsync(
-                            async () =>
-                            {
-                                await dedupManifestClient.DownloadAsync(options, cancellationToken);
-                            },
-                            maxRetries: 3,
-                            tracer: tracer,
-                            canRetryDelegate: e => true,
-                            context: nameof(DownloadMultipleArtifactsAsync),
-                            cancellationToken: cancellationToken,
-                            continueOnCapturedContext: false);
-                    });
-                // Send results to CustomerIntelligence
-                this.context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineArtifact, record: downloadRecord);
+                        { buildArtifact.Name, DedupIdentifier.Create(buildArtifact.Resource.Data) }
+                    };
+
+                    dedupManifestClients.Add(domainId, (dedupManifestClient, clientTelemetry, artifactDictionary));
+                }
+            }
+
+            foreach(var clientInfo in dedupManifestClients.Values)
+            {
+                using (clientInfo.Telemetry)
+                {
+                    // 2) download to the target path
+                    var options = DownloadDedupManifestArtifactOptions.CreateWithMultiManifestIds(
+                        clientInfo.ArtifactDictionary,
+                        downloadParameters.TargetDirectory,
+                        proxyUri: null,
+                        minimatchPatterns: downloadParameters.MinimatchFilters,
+                        minimatchFilterWithArtifactName: downloadParameters.MinimatchFilterWithArtifactName);
+
+                    PipelineArtifactActionRecord downloadRecord = clientInfo.Telemetry.CreateRecord<PipelineArtifactActionRecord>((level, uri, type) =>
+                        new PipelineArtifactActionRecord(level, uri, type, nameof(DownloadMultipleArtifactsAsync), this.context));
+
+                    await clientInfo.Telemetry.MeasureActionAsync(
+                        record: downloadRecord,
+                        actionAsync: async () =>
+                        {
+                            await AsyncHttpRetryHelper.InvokeVoidAsync(
+                                async () =>
+                                {
+                                    await clientInfo.Client.DownloadAsync(options, cancellationToken);
+                                },
+                                maxRetries: 3,
+                                tracer: tracer,
+                                canRetryDelegate: e => true,
+                                context: nameof(DownloadMultipleArtifactsAsync),
+                                cancellationToken: cancellationToken,
+                                continueOnCapturedContext: false);
+                        });
+                    // Send results to CustomerIntelligence
+                    this.context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineArtifact, record: downloadRecord);
+                }
             }
         }
     }