diff --git a/assets.json b/assets.json index 1048f8e564..c09cd710b1 100644 --- a/assets.json +++ b/assets.json @@ -35,6 +35,18 @@ "version": "", "downloadUrl": "https://vstsagentpackage.azureedge.net/agent//pipelines-agent-osx-x64-.tar.gz" }, + { + "name": "vsts-agent-osx-arm64-.tar.gz", + "platform": "osx-arm64", + "version": "", + "downloadUrl": "https://vstsagentpackage.azureedge.net/agent//vsts-agent-osx-arm64-.tar.gz" + }, + { + "name": "pipelines-agent-osx-arm64-.tar.gz", + "platform": "osx-arm64", + "version": "", + "downloadUrl": "https://vstsagentpackage.azureedge.net/agent//pipelines-agent-osx-arm64-.tar.gz" + }, { "name": "vsts-agent-linux-x64-.tar.gz", "platform": "linux-x64", @@ -76,11 +88,5 @@ "platform": "linux-musl-x64", "version": "", "downloadUrl": "https://vstsagentpackage.azureedge.net/agent//vsts-agent-linux-musl-x64-.tar.gz" - }, - { - "name": "pipelines-agent-linux-musl-x64-.tar.gz", - "platform": "linux-musl-x64", - "version": "", - "downloadUrl": "https://vstsagentpackage.azureedge.net/agent//pipelines-agent-linux-musl-x64-.tar.gz" } ] diff --git a/releaseNote.md b/releaseNote.md index a95e7c30c6..b094608827 100644 --- a/releaseNote.md +++ b/releaseNote.md @@ -82,7 +82,7 @@ See [notes](docs/node6.md) on Node version support for more details. | Windows x64 | [pipelines-agent-win-x64-.zip](https://vstsagentpackage.azureedge.net/agent//pipelines-agent-win-x64-.zip) | | | Windows x86 | [pipelines-agent-win-x86-.zip](https://vstsagentpackage.azureedge.net/agent//pipelines-agent-win-x86-.zip) | | | macOS x64 | [pipelines-agent-osx-x64-.tar.gz](https://vstsagentpackage.azureedge.net/agent//pipelines-agent-osx-x64-.tar.gz) | | -| macOS ARM64 | [pipelines-agent-osx-arm64-.tar.gz](https://vstsagentpackage.azureedge.net/agent//pipelines-agent-osx-x64-.tar.gz) | | +| macOS ARM64 | [pipelines-agent-osx-arm64-.tar.gz](https://vstsagentpackage.azureedge.net/agent//pipelines-agent-osx-arm64-.tar.gz) | | | Linux x64 | [pipelines-agent-linux-x64-.tar.gz](https://vstsagentpackage.azureedge.net/agent//pipelines-agent-linux-x64-.tar.gz) | | | Linux ARM | [pipelines-agent-linux-arm-.tar.gz](https://vstsagentpackage.azureedge.net/agent//pipelines-agent-linux-arm-.tar.gz) | | | Linux ARM64 | [pipelines-agent-linux-arm64-.tar.gz](https://vstsagentpackage.azureedge.net/agent//pipelines-agent-linux-arm64-.tar.gz) | | diff --git a/src/Agent.Listener/Configuration/FeatureFlagProvider.cs b/src/Agent.Listener/Configuration/FeatureFlagProvider.cs index 76da891ccf..6c75b0752f 100644 --- a/src/Agent.Listener/Configuration/FeatureFlagProvider.cs +++ b/src/Agent.Listener/Configuration/FeatureFlagProvider.cs @@ -42,12 +42,13 @@ public async Task GetFeatureFlagAsync(IHostContext context, string var credMgr = context.GetService(); var configManager = context.GetService(); + var agentCertManager = context.GetService(); VssCredentials creds = credMgr.LoadCredentials(); ArgUtil.NotNull(creds, nameof(creds)); AgentSettings settings = configManager.LoadSettings(); - using var vssConnection = VssUtil.CreateConnection(new Uri(settings.ServerUrl), creds, traceWriter); + using var vssConnection = VssUtil.CreateConnection(new Uri(settings.ServerUrl), creds, traceWriter, agentCertManager.SkipServerCertificateValidation); var client = vssConnection.GetClient(); try { diff --git a/src/Agent.Listener/JobDispatcher.cs b/src/Agent.Listener/JobDispatcher.cs index e769ea57a0..a262b4f7cd 100644 --- a/src/Agent.Listener/JobDispatcher.cs +++ b/src/Agent.Listener/JobDispatcher.cs @@ -3,6 +3,7 @@ using Agent.Sdk.Knob; using Agent.Sdk.Util; +using Agent.Listener.Configuration; using System; using System.Collections.Concurrent; using System.Collections.Generic; @@ -17,7 +18,9 @@ using System.Linq; using Microsoft.VisualStudio.Services.Common; using System.Diagnostics; -using Agent.Listener.Configuration; +using Newtonsoft.Json; +using Microsoft.VisualStudio.Services.Agent.Listener.Telemetry; + namespace Microsoft.VisualStudio.Services.Agent.Listener { @@ -610,6 +613,13 @@ await processChannel.SendAsync( var messageType = MessageType.CancelRequest; if (HostContext.AgentShutdownToken.IsCancellationRequested) { + var service = HostContext.GetService(); + var ffState = await service.GetFeatureFlagAsync(HostContext, "DistributedTask.Agent.FailJobWhenAgentDies", Trace); + if (ffState.EffectiveState == "On") + { + await PublishTelemetry(message, TaskResult.Failed.ToString(), "100"); + resultOnAbandonOrCancel = TaskResult.Failed; + } switch (HostContext.AgentShutdownReason) { case ShutdownReason.UserCancelled: @@ -915,6 +925,33 @@ private async Task LogWorkerProcessUnhandledException(Pipelines.AgentJobRequestM } } + private async Task PublishTelemetry(Pipelines.AgentJobRequestMessage message, string Task_Result, string TracePoint) + { + try + { + var telemetryData = new Dictionary + { + { "JobId", message.JobId.ToString()}, + { "JobResult", Task_Result }, + { "TracePoint", TracePoint}, + }; + var cmd = new Command("telemetry", "publish") + { + Data = JsonConvert.SerializeObject(telemetryData) + }; + cmd.Properties.Add("area", "PipelinesTasks"); + cmd.Properties.Add("feature", "AgentShutdown"); + + var telemetryPublisher = HostContext.GetService(); + + await telemetryPublisher.PublishEvent(HostContext, cmd); + } + catch (Exception ex) + { + Trace.Warning($"Unable to publish agent shutdown telemetry data. Exception: {ex}"); + } + } + private class WorkerDispatcher : IDisposable { public long RequestId { get; } diff --git a/src/Agent.Listener/Telemetry/TelemetryPublisher.cs b/src/Agent.Listener/Telemetry/TelemetryPublisher.cs index bb1b2fc847..7af372b05d 100644 --- a/src/Agent.Listener/Telemetry/TelemetryPublisher.cs +++ b/src/Agent.Listener/Telemetry/TelemetryPublisher.cs @@ -77,8 +77,9 @@ public async Task PublishEvent(IHostContext context, Command command) var configManager = context.GetService(); AgentSettings settings = configManager.LoadSettings(); + var agentCertManager = context.GetService(); - using var vsConnection = VssUtil.CreateConnection(new Uri(settings.ServerUrl), creds, Trace); + using var vsConnection = VssUtil.CreateConnection(new Uri(settings.ServerUrl), creds, Trace, agentCertManager.SkipServerCertificateValidation); _ciService.Initialize(vsConnection); await PublishEventsAsync(context, ciEvent); diff --git a/src/Agent.Sdk/Knob/AgentKnobs.cs b/src/Agent.Sdk/Knob/AgentKnobs.cs index 35dc100b2c..ff5dff145e 100644 --- a/src/Agent.Sdk/Knob/AgentKnobs.cs +++ b/src/Agent.Sdk/Knob/AgentKnobs.cs @@ -541,6 +541,13 @@ public class AgentKnobs new EnvironmentKnobSource("AZP_AGENT_IGNORE_VSTSTASKLIB"), new BuiltInDefaultKnobSource("false")); + public static readonly Knob FailJobWhenAgentDies = new Knob( + nameof(FailJobWhenAgentDies), + "Mark the Job as Failed instead of Canceled when the Agent dies due to User Cancellation or Shutdown", + new RuntimeKnobSource("FAIL_JOB_WHEN_AGENT_DIES"), + new EnvironmentKnobSource("FAIL_JOB_WHEN_AGENT_DIES"), + new BuiltInDefaultKnobSource("false")); + public static readonly Knob AllowWorkDirectoryRepositories = new Knob( nameof(AllowWorkDirectoryRepositories), "Allows repositories to be checked out below work directory level on self hosted agents.", diff --git a/src/Agent.Worker/JobExtension.cs b/src/Agent.Worker/JobExtension.cs index 4ef648dbfa..ae0f0ea0d4 100644 --- a/src/Agent.Worker/JobExtension.cs +++ b/src/Agent.Worker/JobExtension.cs @@ -13,6 +13,8 @@ using System.Diagnostics; using Agent.Sdk; using Agent.Sdk.Knob; +using Newtonsoft.Json; +using Microsoft.VisualStudio.Services.Agent.Worker.Telemetry; namespace Microsoft.VisualStudio.Services.Agent.Worker { @@ -482,10 +484,22 @@ public async Task> InitializeJob(IExecutionContext jobContext, Pipel catch (OperationCanceledException ex) when (jobContext.CancellationToken.IsCancellationRequested) { // Log the exception and cancel the JobExtension Initialization. - Trace.Error($"Caught cancellation exception from JobExtension Initialization: {ex}"); - context.Error(ex); - context.Result = TaskResult.Canceled; - throw; + if (AgentKnobs.FailJobWhenAgentDies.GetValue(jobContext).AsBoolean() && + HostContext.AgentShutdownToken.IsCancellationRequested) + { + PublishTelemetry(jobContext, TaskResult.Failed.ToString(), "110"); + Trace.Error($"Caught Agent Shutdown exception from JobExtension Initialization: {ex.Message}"); + context.Error(ex); + context.Result = TaskResult.Failed; + throw; + } + else + { + Trace.Error($"Caught cancellation exception from JobExtension Initialization: {ex}"); + context.Error(ex); + context.Result = TaskResult.Canceled; + throw; + } } catch (Exception ex) { @@ -662,6 +676,31 @@ private void OutputSetupInfo(IExecutionContext context) Trace.Error(ex); } } + + private void PublishTelemetry(IExecutionContext context, string Task_Result, string TracePoint) + { + try + { + var telemetryData = new Dictionary + { + { "JobId", context.Variables.System_JobId.ToString()}, + { "JobResult", Task_Result }, + { "TracePoint", TracePoint}, + }; + var cmd = new Command("telemetry", "publish"); + cmd.Data = JsonConvert.SerializeObject(telemetryData, Formatting.None); + cmd.Properties.Add("area", "PipelinesTasks"); + cmd.Properties.Add("feature", "AgentShutdown"); + + var publishTelemetryCmd = new TelemetryCommandExtension(); + publishTelemetryCmd.Initialize(HostContext); + publishTelemetryCmd.ProcessCommand(context, cmd); + } + catch (Exception ex) + { + Trace.Warning($"Unable to publish agent shutdown telemetry data. Exception: {ex}"); + } + } } public class UnsupportedOsException : Exception diff --git a/src/Agent.Worker/JobRunner.cs b/src/Agent.Worker/JobRunner.cs index a052b585ae..d363a5f377 100644 --- a/src/Agent.Worker/JobRunner.cs +++ b/src/Agent.Worker/JobRunner.cs @@ -18,6 +18,8 @@ using System.Threading.Tasks; using System.Net.Http; using Newtonsoft.Json.Linq; +using Newtonsoft.Json; +using Microsoft.VisualStudio.Services.Agent.Worker.Telemetry; namespace Microsoft.VisualStudio.Services.Agent.Worker { @@ -297,9 +299,20 @@ public async Task RunAsync(Pipelines.AgentJobRequestMessage message, { // set the job to canceled // don't log error issue to job ExecutionContext, since server owns the job level issue - Trace.Error($"Job is canceled during initialize."); - Trace.Error($"Caught exception: {ex}"); - return await CompleteJobAsync(jobServer, jobContext, message, TaskResult.Canceled); + if (AgentKnobs.FailJobWhenAgentDies.GetValue(jobContext).AsBoolean() && + HostContext.AgentShutdownToken.IsCancellationRequested) + { + PublishTelemetry(jobContext, TaskResult.Failed.ToString(), "111"); + Trace.Error($"Job is canceled during initialize."); + Trace.Error($"Caught exception: {ex}"); + return await CompleteJobAsync(jobServer, jobContext, message, TaskResult.Failed); + } + else + { + Trace.Error($"Job is canceled during initialize."); + Trace.Error($"Caught exception: {ex}"); + return await CompleteJobAsync(jobServer, jobContext, message, TaskResult.Canceled); + } } catch (Exception ex) { @@ -597,5 +610,30 @@ private void ReplaceConfigUriBaseInJobRequestMessage(Pipelines.AgentJobRequestMe Trace.Info($"Ensure System.TFServerUrl match config url base. {message.Variables[Constants.Variables.System.TFServerUrl].Value}"); } } + + private void PublishTelemetry(IExecutionContext context, string Task_Result, string TracePoint) + { + try + { + var telemetryData = new Dictionary + { + { "JobId", context.Variables.System_JobId.ToString()}, + { "JobResult", Task_Result }, + { "TracePoint", TracePoint}, + }; + var cmd = new Command("telemetry", "publish"); + cmd.Data = JsonConvert.SerializeObject(telemetryData, Formatting.None); + cmd.Properties.Add("area", "PipelinesTasks"); + cmd.Properties.Add("feature", "AgentShutdown"); + + var publishTelemetryCmd = new TelemetryCommandExtension(); + publishTelemetryCmd.Initialize(HostContext); + publishTelemetryCmd.ProcessCommand(context, cmd); + } + catch (Exception ex) + { + Trace.Warning($"Unable to publish agent shutdown telemetry data. Exception: {ex}"); + } + } } } diff --git a/src/Agent.Worker/StepsRunner.cs b/src/Agent.Worker/StepsRunner.cs index 7000f85f2a..6a682e30b4 100644 --- a/src/Agent.Worker/StepsRunner.cs +++ b/src/Agent.Worker/StepsRunner.cs @@ -7,6 +7,7 @@ using System.Threading.Tasks; using Agent.Sdk; +using Agent.Sdk.Knob; using Microsoft.TeamFoundation.DistributedTask.Expressions; using Microsoft.TeamFoundation.DistributedTask.WebApi; @@ -14,6 +15,9 @@ using Pipelines = Microsoft.TeamFoundation.DistributedTask.Pipelines; +using Newtonsoft.Json; +using Microsoft.VisualStudio.Services.Agent.Worker.Telemetry; + namespace Microsoft.VisualStudio.Services.Agent.Worker { public interface IStep @@ -111,6 +115,12 @@ public async Task RunAsync(IExecutionContext jobContext, IList steps) ConditionResult conditionReTestResult; if (HostContext.AgentShutdownToken.IsCancellationRequested) { + if (AgentKnobs.FailJobWhenAgentDies.GetValue(jobContext).AsBoolean()) + { + PublishTelemetry (jobContext, TaskResult.Failed.ToString(), "120"); + jobContext.Result = TaskResult.Failed; + jobContext.Variables.Agent_JobStatus = jobContext.Result; + } step.ExecutionContext.Debug($"Skip Re-evaluate condition on agent shutdown."); conditionReTestResult = false; } @@ -137,6 +147,17 @@ public async Task RunAsync(IExecutionContext jobContext, IList steps) } }); } + else if (AgentKnobs.FailJobWhenAgentDies.GetValue(jobContext).AsBoolean() && + HostContext.AgentShutdownToken.IsCancellationRequested) + { + if (jobContext.Result != TaskResult.Failed) + { + // mark job as failed + PublishTelemetry (jobContext, jobContext.Result.ToString(), "121"); + jobContext.Result = TaskResult.Failed; + jobContext.Variables.Agent_JobStatus = jobContext.Result; + } + } else { if (jobContext.Result != TaskResult.Canceled) @@ -250,6 +271,14 @@ private async Task RunStepAsync(IStep step, CancellationToken jobCancellationTok step.ExecutionContext.Error(StringUtil.Loc("StepTimedOut")); step.ExecutionContext.Result = TaskResult.Failed; } + else if (AgentKnobs.FailJobWhenAgentDies.GetValue(step.ExecutionContext).AsBoolean() && + HostContext.AgentShutdownToken.IsCancellationRequested) + { + PublishTelemetry (step.ExecutionContext, TaskResult.Failed.ToString(), "122"); + Trace.Error($"Caught Agent Shutdown exception from step: {ex.Message}"); + step.ExecutionContext.Error(ex); + step.ExecutionContext.Result = TaskResult.Failed; + } else { // Log the exception and cancel the step. @@ -286,6 +315,16 @@ private async Task RunStepAsync(IStep step, CancellationToken jobCancellationTok // if the step already canceled, don't set it to failed. step.ExecutionContext.CommandResult = TaskResultUtil.MergeTaskResults(step.ExecutionContext.CommandResult, TaskResult.Failed); } + else if (AgentKnobs.FailJobWhenAgentDies.GetValue(step.ExecutionContext).AsBoolean() && + HostContext.AgentShutdownToken.IsCancellationRequested) + { + PublishTelemetry (step.ExecutionContext, TaskResult.Failed.ToString(), "123"); + Trace.Error($"Caught Agent shutdown exception from async command {command.Name}: {ex}"); + step.ExecutionContext.Error(ex); + + // if the step already canceled, don't set it to failed. + step.ExecutionContext.CommandResult = TaskResultUtil.MergeTaskResults(step.ExecutionContext.CommandResult, TaskResult.Failed); + } else { // log and save the OperationCanceledException, set step result to canceled if the current result is not failed. @@ -370,5 +409,30 @@ private async Task SwitchToUtf8Codepage(IStep step) Trace.Warning($"'chcp 65001' failed with exception {ex.Message}"); } } + + private void PublishTelemetry(IExecutionContext context, string Task_Result, string TracePoint) + { + try + { + var telemetryData = new Dictionary + { + { "JobId", context.Variables.System_JobId.ToString()}, + { "JobResult", Task_Result }, + { "TracePoint", TracePoint}, + }; + var cmd = new Command("telemetry", "publish"); + cmd.Data = JsonConvert.SerializeObject(telemetryData, Formatting.None); + cmd.Properties.Add("area", "PipelinesTasks"); + cmd.Properties.Add("feature", "AgentShutdown"); + + var publishTelemetryCmd = new TelemetryCommandExtension(); + publishTelemetryCmd.Initialize(HostContext); + publishTelemetryCmd.ProcessCommand(context, cmd); + } + catch (Exception ex) + { + Trace.Warning($"Unable to publish agent shutdown telemetry data. Exception: {ex}"); + } + } } } diff --git a/src/Common.props b/src/Common.props index edb0e67ed3..ea281c6875 100644 --- a/src/Common.props +++ b/src/Common.props @@ -37,9 +37,12 @@ X86 - + X64 + + ARM64 + X64 diff --git a/src/Misc/Publish.template.ps1 b/src/Misc/Publish.template.ps1 index 45f554bdd6..8d0f3695e5 100644 --- a/src/Misc/Publish.template.ps1 +++ b/src/Misc/Publish.template.ps1 @@ -1,6 +1,6 @@ $ErrorActionPreference = 'Stop' -if ($pwd -notlike '*tfsgheus20' ) { +if ($pwd -notlike '*tfsgheus20') { # primary packages