Users/ivanduplenskikh/add azcli downgrading script

src/Agent.Listener/Configuration/FeatureFlagProvider.cs

@@ -43,18 +43,32 @@ public async Task<FeatureFlag> GetFeatureFlagAsync(IHostContext context, string
 
             VssCredentials creds = credMgr.LoadCredentials();
             ArgUtil.NotNull(creds, nameof(creds));
-            
+
             AgentSettings settings = configManager.LoadSettings();
             using var vssConnection = VssUtil.CreateConnection(new Uri(settings.ServerUrl), creds, traceWriter);
             var client = vssConnection.GetClient<FeatureAvailabilityHttpClient>();
             try
             {
+                // if feature flag doesn't exist, that is traced as an error and logged to Agent log like:
+                // [2023-10-06 22:20:15Z ERR  VisualStudioServices] GET request to https://.../_apis/FeatureFlags/FFName failed. HTTP Status: NotFound, [...]
+                // 
+                // The problem is that we surface that in the console as: 
+                // Error reported in diagnostic logs. Please examine the log for more details.
+                // - /path/to/logs/_diag/Agent_20231006-231735-utc.log
+                //
+                // Reporting an error could cause confusion. Added a flag to the listener so we can temporarily intercept the trace error and convert to warning
+                HostTraceListener.ErrorsAsWarnings = true;
                 return await client.GetFeatureFlagByNameAsync(featureFlagName);
-            } catch(VssServiceException e)
+            }
+            catch (VssServiceException e)
             {
                 Trace.Warning("Unable to retrive feature flag status: " + e.ToString());
                 return new FeatureFlag(featureFlagName, "", "", "Off", "Off");
             }
+            finally
+            {
+                HostTraceListener.ErrorsAsWarnings = false;
+            }
         }
     }
 }

src/Agent.Listener/JobDispatcher.cs

@@ -17,7 +17,7 @@
 using System.Linq;
 using Microsoft.VisualStudio.Services.Common;
 using System.Diagnostics;
-
+using Agent.Listener.Configuration;
 
 namespace Microsoft.VisualStudio.Services.Agent.Listener
 {
@@ -88,6 +88,19 @@ public void Run(Pipelines.AgentJobRequestMessage jobRequestMessage, bool runOnce
                 }
             }
 
+            var service = HostContext.GetService<IFeatureFlagProvider>();
+            string ffState;
+            try
+            {
+                ffState = service.GetFeatureFlagAsync(HostContext, "DistributedTask.Agent.ForceAZCLIToolDowngradeTo252", Trace)?.Result?.EffectiveState;
+            }
+            catch (Exception)
+            {
+                ffState = "Off";
+            }
+
+            jobRequestMessage.Variables[Constants.Variables.Features.ForceAZCLIToolDowngradeTo252] = (ffState == "On").ToString();
+
             WorkerDispatcher newDispatch = new WorkerDispatcher(jobRequestMessage.JobId, jobRequestMessage.RequestId);
             if (runOnce)
             {

src/Agent.Sdk/Knob/AgentKnobs.cs

@@ -511,5 +511,12 @@ public class AgentKnobs
             new RuntimeKnobSource("AZP_AGENT_IGNORE_VSTSTASKLIB"),
             new EnvironmentKnobSource("AZP_AGENT_IGNORE_VSTSTASKLIB"),
             new BuiltInDefaultKnobSource("false"));
+
+        public static readonly Knob ForceAZCLIToolDowngradeTo252 = new Knob(
+            nameof(ForceAZCLIToolDowngradeTo252),
+            "Force downgrading AZ cli to 2.52",
+            new EnvironmentKnobSource("AZP_FORCE_AZCLI_TOOL_DOWNGRADE_TO_2_52"),
+            new RuntimeKnobSource("AZP_FORCE_AZCLI_TOOL_DOWNGRADE_TO_2_52"),
+            new BuiltInDefaultKnobSource("false"));
     }
 }

src/Agent.Worker/JobExtension.cs

@@ -13,6 +13,7 @@
 using System.Diagnostics;
 using Agent.Sdk;
 using Agent.Sdk.Knob;
+using System.Threading;
 
 namespace Microsoft.VisualStudio.Services.Agent.Worker
 {
@@ -228,6 +229,89 @@ public async Task<List<IStep>> InitializeJob(IExecutionContext jobContext, Pipel
                     }
                     context.Output("Finished checking job knob settings.");
 
+                    if (AgentKnobs.ForceAZCLIToolDowngradeTo252.GetValue(jobContext).AsBoolean()
+                        || context.Variables.Get(Constants.Variables.Features.ForceAZCLIToolDowngradeTo252) == "On")
+                    {
+
+                        if (PlatformUtil.RunningOnLinux)
+                        {
+                            context.Output("Temporary step with AZ CLI downgrading.");
+                            var downgradeAZCLIScript = GenerateAZCLIDowngradeScript();
+
+                            context.Output($"temporary file azcli downgrade script: {downgradeAZCLIScript}");
+
+                            using (var processInvoker = HostContext.CreateService<IProcessInvoker>())
+                            {
+                                processInvoker.OutputDataReceived += new EventHandler<ProcessDataReceivedEventArgs>((sender, args) => context.Output(args.Data));
+                                processInvoker.ErrorDataReceived += new EventHandler<ProcessDataReceivedEventArgs>((sender, args) => context.Error(args.Data));
+
+                                int exitCode = 1;
+
+                                if (PlatformUtil.RunningOnWindows)
+                                {
+                                    string powershell = WhichUtil.Which("powershell", require: true, trace: Trace);
+                                    exitCode = await processInvoker.ExecuteAsync(workingDirectory: string.Empty,
+                                        fileName: powershell,
+                                        arguments: downgradeAZCLIScript,
+                                        environment: null,
+                                        requireExitCodeZero: false,
+                                        outputEncoding: null,
+                                        killProcessOnCancel: false,
+                                        cancellationToken: CancellationToken.None);
+                                }
+                                else
+                                {
+                                    string bash = WhichUtil.Which("bash", require: true, trace: Trace);
+                                    exitCode = await processInvoker.ExecuteAsync(
+                                        workingDirectory: string.Empty,
+                                        fileName: bash,
+                                        arguments: downgradeAZCLIScript,
+                                        environment: null,
+                                        cancellationToken: CancellationToken.None);
+                                }
+
+                                if (exitCode == 0)
+                                {
+                                    context.Output($"AZ CLI downgrading installation is finished exit code: {exitCode}");
+                                }
+                                else
+                                {
+                                    throw new Exception($"AZ CLI downgrading installation is finished exit code: {exitCode}");
+                                }
+                            }
+
+                            context.Output($"{downgradeAZCLIScript} is updated in the background.");
+
+                            using (var processInvoker = HostContext.CreateService<IProcessInvoker>())
+                            {
+                                processInvoker.OutputDataReceived += new EventHandler<ProcessDataReceivedEventArgs>((sender, args) => context.Output(args.Data));
+                                processInvoker.ErrorDataReceived += new EventHandler<ProcessDataReceivedEventArgs>((sender, args) => context.Error(args.Data));
+
+                                var exitCode = await processInvoker.ExecuteAsync(
+                                workingDirectory: string.Empty,
+                                fileName: "az",
+                                arguments: "--version",
+                                environment: null,
+                                cancellationToken: CancellationToken.None);
+
+                                if (exitCode == 0)
+                                {
+                                    context.Output($"AZ CLI version exit code: {exitCode}");
+                                }
+                                else
+                                {
+                                    throw new Exception($"AZ CLI version exit code: {exitCode}");
+                                }
+                            }
+
+                            context.Output("Temporary step with AZ CLI finished.");
+                        }
+                    } 
+                    else
+                    {
+                        context.Output("Skip: Temporary step with AZ CLI downgrading due to not running on Windows.");
+                    }
+
                     if (PlatformUtil.RunningOnWindows)
                     {
                         // This is for internal testing and is not publicly supported. This will be removed from the agent at a later time.
@@ -494,6 +578,31 @@ public async Task<List<IStep>> InitializeJob(IExecutionContext jobContext, Pipel
             }
         }
 
+        private string GenerateAZCLIDowngradeScript()
+        {
+            string agentRoot = HostContext.GetDirectory(WellKnownDirectory.Root);
+            string templateName = PlatformUtil.RunningOnWindows ? "azcli_downgrade.ps1.template" : "azcli_downgrade.sh.template";
+            string templatePath = Path.Combine(agentRoot, "bin", templateName);
+            string template = File.ReadAllText(templatePath);
+            string scriptName = PlatformUtil.RunningOnWindows ? "azcli_downgrade.ps1" : "azcli_downgrade.sh";
+            string downgradeScript = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), scriptName);
+
+            if (File.Exists(downgradeScript))
+            {
+                IOUtil.DeleteFile(downgradeScript);
+            }
+
+            File.WriteAllText(downgradeScript, template);
+
+            if (!PlatformUtil.RunningOnWindows)
+            {
+                var unixUtil = HostContext.CreateService<IUnixUtil>();
+                unixUtil.ChmodAsync("755", downgradeScript).GetAwaiter().GetResult();
+            }
+
+            return downgradeScript;
+        }
+
         public async Task FinalizeJob(IExecutionContext jobContext)
         {
             Trace.Entering();

src/Microsoft.VisualStudio.Services.Agent/Constants.cs

@@ -368,6 +368,7 @@ public static class Features
                 // Keep alphabetical. If you add or remove a variable here, do the same in ReadOnlyVariables
                 //
                 public static readonly string BuildDirectoryClean = "agent.clean.buildDirectory";
+                public static readonly string ForceAZCLIToolDowngradeTo252 = "agent.forceazclitooldowngradeto252";
                 public static readonly string GitLfsSupport = "agent.source.git.lfs";
                 public static readonly string GitShallowDepth = "agent.source.git.shallowFetchDepth";
                 public static readonly string SkipSyncSource = "agent.source.skip";
@@ -539,6 +540,7 @@ public static class Task
                 Common.TestResultsDirectory,
                 // Feature variables
                 Features.BuildDirectoryClean,
+                Features.ForceAZCLIToolDowngradeTo252,
                 Features.GitLfsSupport,
                 Features.GitShallowDepth,
                 Features.SkipSyncSource,

src/Microsoft.VisualStudio.Services.Agent/HostTraceListener.cs

@@ -12,6 +12,11 @@ namespace Microsoft.VisualStudio.Services.Agent
 {
     public sealed class HostTraceListener : TextWriterTraceListener
     {
+        /// <summary>
+        /// Use this sparingly if code being called logs an error that needs to be 'downgraded' to a warning        
+        /// </summary>
+        public static bool ErrorsAsWarnings { get; set; }
+
         public bool DisableConsoleReporting { get; set; }
         private const string _logFileNamingPattern = "{0}_{1:yyyyMMdd-HHmmss}-utc.log";
         private string _logFileDirectory;
@@ -64,6 +69,11 @@ public HostTraceListener(string logFile)
         // There must be some TraceFilter extension class that is missing in this source code.
         public override void TraceEvent(TraceEventCache eventCache, string source, TraceEventType eventType, int id, string message)
         {
+            if (ErrorsAsWarnings && eventType < TraceEventType.Warning)
+            {
+                eventType = TraceEventType.Warning;
+            }
+
             if (Filter != null && !Filter.ShouldTrace(eventCache, source, eventType, id, message, null, null, null))
             {
                 return;

src/Misc/layoutbin/azcli_downgrade.ps1.template

@@ -0,0 +1,5 @@
+$ProgressPreference = 'SilentlyContinue'
+$app = Get-WmiObject -Class Win32_Product -Filter "Name = 'Microsoft Azure CLI (64-bit)'"
+$app.Uninstall()
+Invoke-WebRequest -Uri https://azcliprod.blob.core.windows.net/msi/azure-cli-2.52.0-x64.msi -OutFile .\AzureCLI.msi
+Start-Process msiexec.exe -Wait -ArgumentList '/I AzureCLI.msi /quiet' -Verb RunAs

src/Misc/layoutbin/azcli_downgrade.sh.template

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+sudo -n apt-get update
+sudo -n apt-get install -y ca-certificates curl apt-transport-https lsb-release gnupg
+sudo -n mkdir -p /etc/apt/keyrings
+curl -sLS https://packages.microsoft.com/keys/microsoft.asc |
+    gpg --dearmor |
+    sudo -n tee /etc/apt/keyrings/microsoft.gpg > /dev/null
+sudo -n chmod go+r /etc/apt/keyrings/microsoft.gpg
+AZ_DIST=$(lsb_release -cs)
+echo "deb [arch=`dpkg --print-architecture` signed-by=/etc/apt/keyrings/microsoft.gpg] https://packages.microsoft.com/repos/azure-cli/ $AZ_DIST main" |
+    sudo -n tee /etc/apt/sources.list.d/azure-cli.list
+AZ_VER=2.52.0
+sudo -n apt-get update
+sudo -n apt-get install -y azure-cli=$AZ_VER-1~$AZ_DIST --allow-downgrades