[DRAFT] PR to diagnose remote mac testing problems.

Make.config

@@ -317,10 +317,10 @@ MIN_TVOS_SIMULATOR_VERSION=15.0
 EXTRA_SIMULATORS=com.apple.pkg.iPhoneSimulatorSDK15_0 com.apple.pkg.AppleTVSimulatorSDK15_0 com.apple.pkg.WatchSimulatorSDK8_0
 
 INCLUDE_IOS=1
-INCLUDE_MAC=1
-INCLUDE_WATCH=1
-INCLUDE_TVOS=1
-INCLUDE_MACCATALYST=1
+INCLUDE_MAC=
+INCLUDE_WATCH=
+INCLUDE_TVOS=
+INCLUDE_MACCATALYST=
 INCLUDE_DEVICE=1
 INCLUDE_DOTNET_WATCHOS=
 INCLUDE_XAMARIN_LEGACY=

Makefile

@@ -110,6 +110,7 @@ else
 	@echo
 endif
 endif
+	$(Q) $(MAKE) -C dotnet shutdown-build-server
 
 .PHONY: package release
 package release:

dotnet/Makefile

@@ -550,4 +550,9 @@ all-hook::
 	$(Q) $(MAKE) shutdown-build-server
 
 shutdown-build-server:
-	$(Q) $(DOTNET) build-server shutdown
+	$(Q) echo "Shutting down build servers:"
+	$(Q) $(DOTNET) build-server shutdown | sed 's/^/    /' || true
+	$(Q) echo "Listing .NET processes still alive:"
+	$(Q) pgrep -lf "^$(DOTNET)" | sed 's/^/    /' || true
+	$(Q) echo "Killing the above mentioned processes."
+	$(Q) pkill -9 -f "^$(DOTNET)" | sed 's/^/    /' || true

dotnet/targets/Xamarin.Shared.Sdk.targets

@@ -869,8 +869,6 @@
 			Condition="'$(IsMacEnabled)' == 'true'"
 			AdditionalArguments="$(_DittoArchitectures)"
 			CopyFromWindows="true"
-			ToolExe="$(DittoExe)"
-			ToolPath="$(DittoPath)"
 			Source="%(_DirectoriesToPublish.SourceDirectory)"
 			Destination="%(_DirectoriesToPublish.TargetDirectory)"
 			TouchDestinationFiles="true"

msbuild/Directory.Build.props

@@ -10,8 +10,12 @@
 			interesting ways. So in order to get the exact version, they're
 			enclosed in brackets.
 
+			Lists of versions can be found here:
+
+			https://dev.azure.com/azure-public/vside/_artifacts/feed/xamarin-impl/NuGet/Xamarin.Messaging.Client/
+
 		-->
-		<MessagingVersion>[2.1.15]</MessagingVersion>
+		<MessagingVersion>[2.1.41-diagnose-cancelled-posts]</MessagingVersion>
 		<HotRestartVersion>[1.1.7]</HotRestartVersion>
 	</PropertyGroup>
 	<Import Project="$(MSBuildThisFileDirectory)../Directory.Build.props" />

msbuild/Messaging/Xamarin.Messaging.Build/Handlers/ExecuteTaskMessageHandler.cs

@@ -1,3 +1,4 @@
+using System;
 using System.Diagnostics;
 using System.IO;
 using System.Threading.Tasks;
@@ -25,6 +26,10 @@ public ExecuteTaskMessageHandler ()
 
 		protected override async Task<ExecuteTaskResult> ExecuteAsync (ExecuteTaskMessage message)
 		{
+			var msg = $"{System.DateTime.UtcNow.ToString ("o")} ExecuteTaskMessageHandler.ExecuteAsync ({message.TaskName})\n{System.Environment.StackTrace}";
+			Console.WriteLine (msg);
+			Console.Error.WriteLine (msg);
+			tracer.Info (msg);
 			return await Task.Run (() => {
 				// We need to lock in order to change the current directory
 				lock (lockObject) {

msbuild/Xamarin.MacDev.Tasks/Decompress.cs

@@ -4,6 +4,7 @@
 using System.IO;
 using System.IO.Compression;
 using System.Reflection;
+using System.Threading;
 
 using Microsoft.Build.Framework;
 using Microsoft.Build.Utilities;
@@ -81,9 +82,10 @@ public static bool IsCompressed (string path)
 		/// <param name="zip">The zip to search in</param>
 		/// <param name="resource">The relative path inside the zip to extract (may be a file or a directory).</param>
 		/// <param name="decompressionDir">The location on disk to store the extracted results</param>
+		/// <param name="cancellationToken">The cancellation token (if any=</param>
 		/// <param name="decompressedResource">The location on disk to the extracted resource</param>
 		/// <returns></returns>
-		public static bool TryDecompress (TaskLoggingHelper log, string zip, string resource, string decompressionDir, List<string> createdFiles, [NotNullWhen (true)] out string? decompressedResource)
+		public static bool TryDecompress (TaskLoggingHelper log, string zip, string resource, string decompressionDir, List<string> createdFiles, CancellationToken? cancellationToken, [NotNullWhen (true)] out string? decompressedResource)
 		{
 			decompressedResource = Path.Combine (decompressionDir, resource);
 
@@ -101,11 +103,11 @@ public static bool TryDecompress (TaskLoggingHelper log, string zip, string reso
 
 			bool rv;
 			if (Environment.OSVersion.Platform == PlatformID.Win32NT) {
-				rv = TryDecompressUsingSystemIOCompression (log, zip, resource, decompressionDir);
+				rv = TryDecompressUsingSystemIOCompression (log, zip, resource, decompressionDir, cancellationToken);
 			} else if (!string.IsNullOrEmpty (Environment.GetEnvironmentVariable ("XAMARIN_USE_SYSTEM_IO_COMPRESSION"))) {
-				rv = TryDecompressUsingSystemIOCompression (log, zip, resource, decompressionDir);
+				rv = TryDecompressUsingSystemIOCompression (log, zip, resource, decompressionDir, cancellationToken);
 			} else {
-				rv = TryDecompressUsingUnzip (log, zip, resource, decompressionDir);
+				rv = TryDecompressUsingUnzip (log, zip, resource, decompressionDir, cancellationToken);
 			}
 
 			if (rv) {
@@ -128,7 +130,7 @@ public static bool TryDecompress (TaskLoggingHelper log, string zip, string reso
 		// The dir separator character in zip files is always "/", even on Windows
 		const char zipDirectorySeparator = '/';
 
-		static bool TryDecompressUsingUnzip (TaskLoggingHelper log, string zip, string resource, string decompressionDir)
+		static bool TryDecompressUsingUnzip (TaskLoggingHelper log, string zip, string resource, string decompressionDir, CancellationToken? cancellationToken)
 		{
 			Directory.CreateDirectory (decompressionDir);
 			var args = new List<string> {
@@ -157,11 +159,11 @@ static bool TryDecompressUsingUnzip (TaskLoggingHelper log, string zip, string r
 				args.Add (zipPattern);
 			}
 
-			var rv = XamarinTask.ExecuteAsync (log, "unzip", args).Result;
+			var rv = XamarinTask.ExecuteAsync (log, "unzip", args, cancellationToken: cancellationToken).Result;
 			return rv.ExitCode == 0;
 		}
 
-		static bool TryDecompressUsingSystemIOCompression (TaskLoggingHelper log, string zip, string resource, string decompressionDir)
+		static bool TryDecompressUsingSystemIOCompression (TaskLoggingHelper log, string zip, string resource, string decompressionDir, CancellationToken? cancellationToken)
 		{
 			var rv = true;
 
@@ -172,6 +174,7 @@ static bool TryDecompressUsingSystemIOCompression (TaskLoggingHelper log, string
 
 			using var archive = ZipFile.OpenRead (zip);
 			foreach (var entry in archive.Entries) {
+				cancellationToken?.ThrowIfCancellationRequested ();
 				var entryPath = entry.FullName;
 				if (entryPath.Length == 0)
 					continue;
@@ -207,7 +210,11 @@ static bool TryDecompressUsingSystemIOCompression (TaskLoggingHelper log, string
 					Directory.CreateDirectory (Path.GetDirectoryName (targetPath));
 					using var streamWrite = File.OpenWrite (targetPath);
 					using var streamRead = entry.Open ();
-					streamRead.CopyTo (streamWrite);
+#if NET
+					streamRead.CopyToAsync (streamWrite, cancellationToken ?? CancellationToken.None).Wait ();
+#else
+					streamRead.CopyToAsync (streamWrite, 81920 /* default buffer size according to docs */, cancellationToken ?? CancellationToken.None).Wait ();
+#endif
 					log.LogMessage (MessageImportance.Low, "Extracted {0} into {1}", entryPath, targetPath);
 				}
 			}

msbuild/Xamarin.MacDev.Tasks/MsBuildTasks/Copy.cs

@@ -13,16 +13,23 @@ public class Copy : Microsoft_Build_Tasks_Core::Microsoft.Build.Tasks.Copy {
 		public string SessionId { get; set; } = string.Empty;
 		public override bool Execute ()
 		{
-			if (!this.ShouldExecuteRemotely (SessionId))
-				return base.Execute ();
+			try {
+				if (!this.ShouldExecuteRemotely (SessionId))
+					return base.Execute ();
 
-			var taskRunner = new TaskRunner (SessionId, BuildEngine4);
+				var taskRunner = new TaskRunner (SessionId, BuildEngine4);
 
-			if (SourceFiles?.Any () == true) {
-				taskRunner.FixReferencedItems (this, SourceFiles);
-			}
+				if (SourceFiles?.Any () == true) {
+					taskRunner.FixReferencedItems (this, SourceFiles);
+				}
 
-			return taskRunner.RunAsync (this).Result;
+				return taskRunner.RunAsync (this).Result;
+			} catch (Exception ex) {
+				Log.LogError ($"Copy failed due to exception: {ex.Message}");
+				Log.LogError ($"Stack trace:\n{ex.StackTrace}");
+				Log.LogErrorFromException (ex);
+				return false;
+			}
 		}
 	}
 }

msbuild/Xamarin.MacDev.Tasks/Tasks/Ditto.cs

@@ -4,14 +4,16 @@
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
+using System.Threading;
 
 using Microsoft.Build.Framework;
 using Microsoft.Build.Utilities;
 
 using Xamarin.Messaging.Build.Client;
+using Xamarin.Utils;
 
 namespace Xamarin.MacDev.Tasks {
-	public class Ditto : XamarinToolTask, ITaskCallback {
+	public class Ditto : XamarinTask, ITaskCallback, ICancelableTask {
 		#region Inputs
 
 		public string? AdditionalArguments { get; set; }
@@ -40,44 +42,56 @@ public class Ditto : XamarinToolTask, ITaskCallback {
 
 		#endregion
 
-		protected override string ToolName {
-			get { return "ditto"; }
-		}
-
-		protected override string GenerateFullPathToTool ()
-		{
-			if (!string.IsNullOrEmpty (ToolPath))
-				return Path.Combine (ToolPath, ToolExe);
-
-			var path = Path.Combine ("/usr/bin", ToolExe);
-
-			return File.Exists (path) ? path : ToolExe;
-		}
+		CancellationTokenSource? cancellationTokenSource;
 
-		protected override string GenerateCommandLineCommands ()
+		void LogLine (string msg)
 		{
-			var args = new CommandLineArgumentBuilder ();
-
-			args.AddQuoted (Path.GetFullPath (Source!.ItemSpec));
-			args.AddQuoted (Path.GetFullPath (Destination!.ItemSpec));
-			if (!string.IsNullOrEmpty (AdditionalArguments))
-				args.Add (AdditionalArguments);
-
-			return args.ToString ();
+			Log.LogWarning (msg);
+			Console.WriteLine ($"{DateTime.UtcNow.ToString ("o")} Ditto stdout: {msg}");
+			Console.Error.WriteLine ($"{DateTime.UtcNow.ToString ("o")} Ditto stderr: {msg}");
 		}
 
 		public override bool Execute ()
 		{
 			if (ShouldExecuteRemotely ()) {
+				LogLine ($"Ditto.Execute () will execute remotely");
 				var taskRunner = new TaskRunner (SessionId, BuildEngine4);
 
 				taskRunner.FixReferencedItems (this, new ITaskItem [] { Source! });
 
-				return taskRunner.RunAsync (this).Result;
+				LogLine ($"Ditto.Execute () about to execute remotely");
+				var rv = taskRunner.RunAsync (this).Result;
+				LogLine ($"Ditto.Execute () executed remotely: {rv}");
+				return rv;
 			}
 
-			if (!base.Execute ())
+			var src = Source!.ItemSpec;
+			if (!File.Exists (src) && !Directory.Exists (src)) {
+				Log.LogError ($"The source {src} does not exist.");
 				return false;
+			}
+
+			var args = new List<string> ();
+			args.Add (Path.GetFullPath (Source!.ItemSpec));
+			args.Add (Path.GetFullPath (Destination!.ItemSpec));
+#if NET
+			if (!string.IsNullOrEmpty (AdditionalArguments)) {
+#else
+			if (AdditionalArguments is not null && !string.IsNullOrEmpty (AdditionalArguments)) {
+#endif
+				if (StringUtils.TryParseArguments (AdditionalArguments, out var additionalArgs, out var ex)) {
+					args.AddRange (additionalArgs);
+				} else {
+					Log.LogError ("Unable to parse the AdditionalArguments: {0}", AdditionalArguments);
+					return false;
+				}
+			}
+
+			LogLine ($"Ditto.Execute () about to execute locally\n{Environment.StackTrace}");
+			cancellationTokenSource = new CancellationTokenSource ();
+			cancellationTokenSource.CancelAfter (TimeSpan.FromMinutes (2)); // FIXME: remove this
+			ExecuteAsync (Log, "/usr/bin/ditto", args, cancellationToken: cancellationTokenSource.Token).Wait ();
+			LogLine ($"Ditto.Execute () executed locally");
 
 			// Create a list of all the files we've copied
 			var copiedFiles = new List<ITaskItem> ();
@@ -96,18 +110,13 @@ public override bool Execute ()
 			return !Log.HasLoggedErrors;
 		}
 
-		protected override void LogEventsFromTextOutput (string singleLine, MessageImportance messageImportance)
-		{
-			// TODO: do proper parsing of error messages and such
-			Log.LogMessage (messageImportance, "{0}", singleLine);
-		}
-
-		public override void Cancel ()
+		public void Cancel ()
 		{
-			base.Cancel ();
-
-			if (ShouldExecuteRemotely ())
+			if (ShouldExecuteRemotely ()) {
 				BuildConnection.CancelAsync (BuildEngine4).Wait ();
+			} else {
+				cancellationTokenSource?.Cancel ();
+			}
 		}
 
 		public IEnumerable<ITaskItem> GetAdditionalItemsToBeCopied ()