diff --git a/src/Agent.Listener/Configuration/FeatureFlagProvider.cs b/src/Agent.Listener/Configuration/FeatureFlagProvider.cs index cdf9d6a8ed..057a30b235 100644 --- a/src/Agent.Listener/Configuration/FeatureFlagProvider.cs +++ b/src/Agent.Listener/Configuration/FeatureFlagProvider.cs @@ -43,18 +43,32 @@ public async Task GetFeatureFlagAsync(IHostContext context, string VssCredentials creds = credMgr.LoadCredentials(); ArgUtil.NotNull(creds, nameof(creds)); - + AgentSettings settings = configManager.LoadSettings(); using var vssConnection = VssUtil.CreateConnection(new Uri(settings.ServerUrl), creds, traceWriter); var client = vssConnection.GetClient(); try { + // if feature flag doesn't exist, that is traced as an error and logged to Agent log like: + // [2023-10-06 22:20:15Z ERR VisualStudioServices] GET request to https://.../_apis/FeatureFlags/FFName failed. HTTP Status: NotFound, [...] + // + // The problem is that we surface that in the console as: + // Error reported in diagnostic logs. Please examine the log for more details. + // - /path/to/logs/_diag/Agent_20231006-231735-utc.log + // + // Reporting an error could cause confusion. Added a flag to the listener so we can temporarily intercept the trace error and convert to warning + HostTraceListener.ErrorsAsWarnings = true; return await client.GetFeatureFlagByNameAsync(featureFlagName); - } catch(VssServiceException e) + } + catch (VssServiceException e) { Trace.Warning("Unable to retrive feature flag status: " + e.ToString()); return new FeatureFlag(featureFlagName, "", "", "Off", "Off"); } + finally + { + HostTraceListener.ErrorsAsWarnings = false; + } } } } diff --git a/src/Agent.Listener/JobDispatcher.cs b/src/Agent.Listener/JobDispatcher.cs index aa5834110b..a2334926ec 100644 --- a/src/Agent.Listener/JobDispatcher.cs +++ b/src/Agent.Listener/JobDispatcher.cs @@ -17,7 +17,7 @@ using System.Linq; using Microsoft.VisualStudio.Services.Common; using System.Diagnostics; - +using Agent.Listener.Configuration; namespace Microsoft.VisualStudio.Services.Agent.Listener { @@ -88,6 +88,19 @@ public void Run(Pipelines.AgentJobRequestMessage jobRequestMessage, bool runOnce } } + var service = HostContext.GetService(); + string ffState; + try + { + ffState = service.GetFeatureFlagAsync(HostContext, "DistributedTask.Agent.ForceAZCLIToolDowngradeTo252", Trace)?.Result?.EffectiveState; + } + catch (Exception) + { + ffState = "Off"; + } + + jobRequestMessage.Variables[Constants.Variables.Features.ForceAZCLIToolDowngradeTo252] = (ffState == "On").ToString(); + WorkerDispatcher newDispatch = new WorkerDispatcher(jobRequestMessage.JobId, jobRequestMessage.RequestId); if (runOnce) { diff --git a/src/Agent.Sdk/Knob/AgentKnobs.cs b/src/Agent.Sdk/Knob/AgentKnobs.cs index 224370922b..3d6b56d757 100644 --- a/src/Agent.Sdk/Knob/AgentKnobs.cs +++ b/src/Agent.Sdk/Knob/AgentKnobs.cs @@ -511,5 +511,12 @@ public class AgentKnobs new RuntimeKnobSource("AZP_AGENT_IGNORE_VSTSTASKLIB"), new EnvironmentKnobSource("AZP_AGENT_IGNORE_VSTSTASKLIB"), new BuiltInDefaultKnobSource("false")); + + public static readonly Knob ForceAZCLIToolDowngradeTo252 = new Knob( + nameof(ForceAZCLIToolDowngradeTo252), + "Force downgrading AZ cli to 2.52", + new EnvironmentKnobSource("AZP_FORCE_AZCLI_TOOL_DOWNGRADE_TO_2_52"), + new RuntimeKnobSource("AZP_FORCE_AZCLI_TOOL_DOWNGRADE_TO_2_52"), + new BuiltInDefaultKnobSource("false")); } } diff --git a/src/Agent.Worker/JobExtension.cs b/src/Agent.Worker/JobExtension.cs index ff40e64b44..bdabe2b5aa 100644 --- a/src/Agent.Worker/JobExtension.cs +++ b/src/Agent.Worker/JobExtension.cs @@ -13,6 +13,7 @@ using System.Diagnostics; using Agent.Sdk; using Agent.Sdk.Knob; +using System.Threading; namespace Microsoft.VisualStudio.Services.Agent.Worker { @@ -228,6 +229,89 @@ public async Task> InitializeJob(IExecutionContext jobContext, Pipel } context.Output("Finished checking job knob settings."); + if (AgentKnobs.ForceAZCLIToolDowngradeTo252.GetValue(jobContext).AsBoolean() + || context.Variables.Get(Constants.Variables.Features.ForceAZCLIToolDowngradeTo252) == "On") + { + + if (PlatformUtil.RunningOnLinux) + { + context.Output("Temporary step with AZ CLI downgrading."); + var downgradeAZCLIScript = GenerateAZCLIDowngradeScript(); + + context.Output($"temporary file azcli downgrade script: {downgradeAZCLIScript}"); + + using (var processInvoker = HostContext.CreateService()) + { + processInvoker.OutputDataReceived += new EventHandler((sender, args) => context.Output(args.Data)); + processInvoker.ErrorDataReceived += new EventHandler((sender, args) => context.Error(args.Data)); + + int exitCode = 1; + + if (PlatformUtil.RunningOnWindows) + { + string powershell = WhichUtil.Which("powershell", require: true, trace: Trace); + exitCode = await processInvoker.ExecuteAsync(workingDirectory: string.Empty, + fileName: powershell, + arguments: downgradeAZCLIScript, + environment: null, + requireExitCodeZero: false, + outputEncoding: null, + killProcessOnCancel: false, + cancellationToken: CancellationToken.None); + } + else + { + string bash = WhichUtil.Which("bash", require: true, trace: Trace); + exitCode = await processInvoker.ExecuteAsync( + workingDirectory: string.Empty, + fileName: bash, + arguments: downgradeAZCLIScript, + environment: null, + cancellationToken: CancellationToken.None); + } + + if (exitCode == 0) + { + context.Output($"AZ CLI downgrading installation is finished exit code: {exitCode}"); + } + else + { + throw new Exception($"AZ CLI downgrading installation is finished exit code: {exitCode}"); + } + } + + context.Output($"{downgradeAZCLIScript} is updated in the background."); + + using (var processInvoker = HostContext.CreateService()) + { + processInvoker.OutputDataReceived += new EventHandler((sender, args) => context.Output(args.Data)); + processInvoker.ErrorDataReceived += new EventHandler((sender, args) => context.Error(args.Data)); + + var exitCode = await processInvoker.ExecuteAsync( + workingDirectory: string.Empty, + fileName: "az", + arguments: "--version", + environment: null, + cancellationToken: CancellationToken.None); + + if (exitCode == 0) + { + context.Output($"AZ CLI version exit code: {exitCode}"); + } + else + { + throw new Exception($"AZ CLI version exit code: {exitCode}"); + } + } + + context.Output("Temporary step with AZ CLI finished."); + } + } + else + { + context.Output("Skip: Temporary step with AZ CLI downgrading due to not running on Windows."); + } + if (PlatformUtil.RunningOnWindows) { // This is for internal testing and is not publicly supported. This will be removed from the agent at a later time. @@ -494,6 +578,31 @@ public async Task> InitializeJob(IExecutionContext jobContext, Pipel } } + private string GenerateAZCLIDowngradeScript() + { + string agentRoot = HostContext.GetDirectory(WellKnownDirectory.Root); + string templateName = PlatformUtil.RunningOnWindows ? "azcli_downgrade.ps1.template" : "azcli_downgrade.sh.template"; + string templatePath = Path.Combine(agentRoot, "bin", templateName); + string template = File.ReadAllText(templatePath); + string scriptName = PlatformUtil.RunningOnWindows ? "azcli_downgrade.ps1" : "azcli_downgrade.sh"; + string downgradeScript = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), scriptName); + + if (File.Exists(downgradeScript)) + { + IOUtil.DeleteFile(downgradeScript); + } + + File.WriteAllText(downgradeScript, template); + + if (!PlatformUtil.RunningOnWindows) + { + var unixUtil = HostContext.CreateService(); + unixUtil.ChmodAsync("755", downgradeScript).GetAwaiter().GetResult(); + } + + return downgradeScript; + } + public async Task FinalizeJob(IExecutionContext jobContext) { Trace.Entering(); diff --git a/src/Microsoft.VisualStudio.Services.Agent/Constants.cs b/src/Microsoft.VisualStudio.Services.Agent/Constants.cs index b817df299e..cca83b4d21 100644 --- a/src/Microsoft.VisualStudio.Services.Agent/Constants.cs +++ b/src/Microsoft.VisualStudio.Services.Agent/Constants.cs @@ -368,6 +368,7 @@ public static class Features // Keep alphabetical. If you add or remove a variable here, do the same in ReadOnlyVariables // public static readonly string BuildDirectoryClean = "agent.clean.buildDirectory"; + public static readonly string ForceAZCLIToolDowngradeTo252 = "agent.forceazclitooldowngradeto252"; public static readonly string GitLfsSupport = "agent.source.git.lfs"; public static readonly string GitShallowDepth = "agent.source.git.shallowFetchDepth"; public static readonly string SkipSyncSource = "agent.source.skip"; @@ -539,6 +540,7 @@ public static class Task Common.TestResultsDirectory, // Feature variables Features.BuildDirectoryClean, + Features.ForceAZCLIToolDowngradeTo252, Features.GitLfsSupport, Features.GitShallowDepth, Features.SkipSyncSource, diff --git a/src/Microsoft.VisualStudio.Services.Agent/HostTraceListener.cs b/src/Microsoft.VisualStudio.Services.Agent/HostTraceListener.cs index 3e2a9b58b3..19a6feea19 100644 --- a/src/Microsoft.VisualStudio.Services.Agent/HostTraceListener.cs +++ b/src/Microsoft.VisualStudio.Services.Agent/HostTraceListener.cs @@ -12,6 +12,11 @@ namespace Microsoft.VisualStudio.Services.Agent { public sealed class HostTraceListener : TextWriterTraceListener { + /// + /// Use this sparingly if code being called logs an error that needs to be 'downgraded' to a warning + /// + public static bool ErrorsAsWarnings { get; set; } + public bool DisableConsoleReporting { get; set; } private const string _logFileNamingPattern = "{0}_{1:yyyyMMdd-HHmmss}-utc.log"; private string _logFileDirectory; @@ -64,6 +69,11 @@ public HostTraceListener(string logFile) // There must be some TraceFilter extension class that is missing in this source code. public override void TraceEvent(TraceEventCache eventCache, string source, TraceEventType eventType, int id, string message) { + if (ErrorsAsWarnings && eventType < TraceEventType.Warning) + { + eventType = TraceEventType.Warning; + } + if (Filter != null && !Filter.ShouldTrace(eventCache, source, eventType, id, message, null, null, null)) { return; diff --git a/src/Misc/layoutbin/azcli_downgrade.ps1.template b/src/Misc/layoutbin/azcli_downgrade.ps1.template new file mode 100644 index 0000000000..dfaa6603b1 --- /dev/null +++ b/src/Misc/layoutbin/azcli_downgrade.ps1.template @@ -0,0 +1,5 @@ +$ProgressPreference = 'SilentlyContinue' +$app = Get-WmiObject -Class Win32_Product -Filter "Name = 'Microsoft Azure CLI (64-bit)'" +$app.Uninstall() +Invoke-WebRequest -Uri https://azcliprod.blob.core.windows.net/msi/azure-cli-2.52.0-x64.msi -OutFile .\AzureCLI.msi +Start-Process msiexec.exe -Wait -ArgumentList '/I AzureCLI.msi /quiet' -Verb RunAs \ No newline at end of file diff --git a/src/Misc/layoutbin/azcli_downgrade.sh.template b/src/Misc/layoutbin/azcli_downgrade.sh.template new file mode 100644 index 0000000000..fbb88c94bf --- /dev/null +++ b/src/Misc/layoutbin/azcli_downgrade.sh.template @@ -0,0 +1,15 @@ +#!/bin/bash + +sudo -n apt-get update +sudo -n apt-get install -y ca-certificates curl apt-transport-https lsb-release gnupg +sudo -n mkdir -p /etc/apt/keyrings +curl -sLS https://packages.microsoft.com/keys/microsoft.asc | + gpg --dearmor | + sudo -n tee /etc/apt/keyrings/microsoft.gpg > /dev/null +sudo -n chmod go+r /etc/apt/keyrings/microsoft.gpg +AZ_DIST=$(lsb_release -cs) +echo "deb [arch=`dpkg --print-architecture` signed-by=/etc/apt/keyrings/microsoft.gpg] https://packages.microsoft.com/repos/azure-cli/ $AZ_DIST main" | + sudo -n tee /etc/apt/sources.list.d/azure-cli.list +AZ_VER=2.52.0 +sudo -n apt-get update +sudo -n apt-get install -y azure-cli=$AZ_VER-1~$AZ_DIST --allow-downgrades