Revert "[msbuild/tests] Use 'dotnet test' for the MSBuild tests. (#21263)"

msbuild/ILMerge.targets

@@ -2,7 +2,7 @@
 <Project xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
   <PropertyGroup>
     <CopyLocalLockFileAssemblies>true</CopyLocalLockFileAssemblies>
-    <MergeSystemAssemblies Condition="'$(MergeSystemAssemblies)' == ''">false</MergeSystemAssemblies>
+    <MergeSystemAssemblies Condition="'$(MergeSystemAssemblies)' == ''">true</MergeSystemAssemblies>
   </PropertyGroup>
 
   <ItemGroup>

msbuild/Xamarin.MacDev.Tasks/Tasks/ComputeCodesignItems.cs

@@ -161,7 +161,7 @@ public override bool Execute ()
 				for (var i = 1; i < all.Length; i++) {
 					var nextMetadata = all [i].CloneCustomMetadataToDictionary ();
 					if (nextMetadata.Count != firstMetadata.Count) {
-						Log.LogWarning (MSBStrings.W7095, /* Code signing has been requested multiple times for '{0}', with different metadata. The metadata for one are: '{1}', while the metadata for the other are: '{2}' */ group.Key, string.Join (", ", firstMetadata.Keys.OrderBy (v => v)), string.Join (", ", nextMetadata.Keys.OrderBy (v => v)));
+						Log.LogWarning (MSBStrings.W7095, /* Code signing has been requested multiple times for '{0}', with different metadata. The metadata for one are: '{1}', while the metadata for the other are: '{2}' */ group.Key, string.Join (", ", firstMetadata.Keys), string.Join (", ", nextMetadata.Keys));
 					} else {
 						foreach (var kvp in firstMetadata) {
 							if (!nextMetadata.TryGetValue (kvp.Key, out var nextValue)) {

tests/Makefile

@@ -217,19 +217,15 @@ killall:
 NUNIT_MSBUILD_DIR=$(TOP)/packages/NUnit.Runners.2.6.4/tools/lib
 test-ios-tasks: test-macdev-tests test-macdev-tasks
 
-# Example TEST_FILTER:
-#    TEST_FILTER="--filter FullyQualifiedName~BuildMyCocoaApp"
-# Docs: https://docs.microsoft.com/en-us/dotnet/core/tools/dotnet-test#filter-option-details
 test-macdev-tests: export MSBUILD_EXE_PATH=
 test-macdev-tests: verify-system-vsmac-xcode-match
-	$(Q) $(DOTNET) test $(TOP)/tests/msbuild/Xamarin.MacDev.Tests/Xamarin.MacDev.Tests.csproj $(TEST_FILTER)
+	$(Q) $(DOTNET) build "/bl:$@.binlog" $(TOP)/tests/msbuild/Xamarin.MacDev.Tests/Xamarin.MacDev.Tests.csproj /p:Configuration=Debug $(DOTNET_BUILD_VERBOSITY)
+	cd $(TOP)/tests/msbuild/Xamarin.MacDev.Tests && $(SYSTEM_XIBUILD) -t -- $(abspath $(TOP)/tools/nunit3-console-3.11.1) $(abspath $(TOP)/tests/msbuild/Xamarin.MacDev.Tests/bin/Debug/net472/Xamarin.MacDev.Tests.dll) "--result=$(abspath $(CURDIR)/TestResults_Xamarin.MacDev.Tests.xml);format=nunit2" -labels=After $(TEST_FIXTURE)
 
-# Example TEST_FILTER:
-#    TEST_FILTER="--filter FullyQualifiedName~BuildMyCocoaApp"
-# Docs: https://docs.microsoft.com/en-us/dotnet/core/tools/dotnet-test#filter-option-details
 test-macdev-tasks: export MSBUILD_EXE_PATH=
 test-macdev-tasks: verify-system-vsmac-xcode-match
-	$(Q) $(DOTNET) test $(TOP)/tests/msbuild/Xamarin.MacDev.Tasks.Tests/Xamarin.MacDev.Tasks.Tests.csproj $(TEST_FILTER)
+	$(Q) $(DOTNET) build "/bl:$@.binlog" $(TOP)/tests/msbuild/Xamarin.MacDev.Tasks.Tests/Xamarin.MacDev.Tasks.Tests.csproj /p:Configuration=Debug $(DOTNET_BUILD_VERBOSITY)
+	cd $(TOP)/tests/msbuild/Xamarin.MacDev.Tasks.Tests && $(SYSTEM_XIBUILD) -t -- $(abspath $(TOP)/tools/nunit3-console-3.11.1) $(abspath $(TOP)/tests/msbuild/Xamarin.MacDev.Tasks.Tests/bin/Debug/net472/Xamarin.MacDev.Tasks.Tests.dll) "--result=$(abspath $(CURDIR)/TestResults_Xamarin.MacDev.Tasks.Tests.xml)" -labels=After $(TEST_FIXTURE)
 
 mac-test-package.zip:
 ifdef INCLUDE_MAC

tests/msbuild/Xamarin.MacDev.Tasks.Tests/TaskTests/CompileAppManifestTaskTests.cs

@@ -21,8 +21,8 @@ CompileAppManifest CreateTask (string? tmpdir = null, ApplePlatform platform = A
 			task.AssemblyName = "AssemblyName";
 			task.AppBundleName = "AppBundleName";
 			task.CompiledAppManifest = new TaskItem (Path.Combine (tmpdir, "TemporaryAppManifest.plist"));
-			task.DefaultSdkVersion = Sdks.GetAppleSdk (platform).GetInstalledSdkVersions (false).First ().ToString ()!;
-			task.SdkVersion = task.DefaultSdkVersion ?? string.Empty;
+			task.DefaultSdkVersion = Sdks.GetAppleSdk (platform).GetInstalledSdkVersions (false).First ().ToString ();
+			task.SdkVersion = task.DefaultSdkVersion;
 			task.TargetFrameworkMoniker = TargetFramework.GetTargetFramework (platform, true).ToString ();
 
 			return task;

tests/msbuild/Xamarin.MacDev.Tasks.Tests/TaskTests/CompileEntitlementsTaskTests.cs

@@ -34,8 +34,8 @@ CustomCompileEntitlements CreateEntitlementsTask (out string compiledEntitlement
 			task.AppBundleDir = AppBundlePath;
 			task.BundleIdentifier = "com.xamarin.MySingleView";
 			task.CompiledEntitlements = new TaskItem (Path.Combine (MonoTouchProjectObjPath, "Entitlements.xcent"));
-			task.Entitlements = Path.Combine (Path.GetDirectoryName (GetType ().Assembly.Location)!, "Resources", "Entitlements.plist");
-			task.ProvisioningProfile = Path.Combine (Path.GetDirectoryName (GetType ().Assembly.Location)!, "Resources", "profile.mobileprovision");
+			task.Entitlements = Path.Combine (Path.GetDirectoryName (GetType ().Assembly.Location), "Resources", "Entitlements.plist");
+			task.ProvisioningProfile = Path.Combine (Path.GetDirectoryName (GetType ().Assembly.Location), "Resources", "profile.mobileprovision");
 			task.SdkPlatform = "iPhoneOS";
 			task.SdkVersion = "6.1";
 			task.TargetFrameworkMoniker = "Xamarin.iOS,v1.0";

tests/msbuild/Xamarin.MacDev.Tasks.Tests/TaskTests/ComputeCodesignItemsTaskTests.cs

@@ -496,7 +496,7 @@ public void DuplicatedWithDifferentMetadata (ApplePlatform platform, bool isDotN
 				Assert.AreEqual (3, Engine.Logger.WarningsEvents.Count, "Warning Count");
 				Assert.AreEqual ("Code signing has been requested multiple times for 'Bundle.app/Contents/MonoBundle/createdump', with different metadata. The metadata 'OnlyIn1=true' has been set for one item, but not the other.", Engine.Logger.WarningsEvents [0].Message, "Message #0");
 				Assert.AreEqual ("Code signing has been requested multiple times for 'Bundle.app/Contents/MonoBundle/createdump', with different metadata. The metadata 'InOneAndTwoWithDifferentValues' has different values for each item (once it's '1', another time it's '2').", Engine.Logger.WarningsEvents [1].Message, "Message #1");
-				Assert.AreEqual ("Code signing has been requested multiple times for 'Bundle.app/Contents/MonoBundle/createdump', with different metadata. The metadata for one are: 'CodesignStampFile, InOneAndTwoWithDifferentValues, OnlyIn1, RequireCodeSigning', while the metadata for the other are: 'CodesignStampFile, RequireCodeSigning'", Engine.Logger.WarningsEvents [2].Message, "Message #2");
+				Assert.AreEqual ("Code signing has been requested multiple times for 'Bundle.app/Contents/MonoBundle/createdump', with different metadata. The metadata for one are: 'RequireCodeSigning, OnlyIn1, InOneAndTwoWithDifferentValues, CodesignStampFile', while the metadata for the other are: 'RequireCodeSigning, CodesignStampFile'", Engine.Logger.WarningsEvents [2].Message, "Message #2");
 
 				VerifyCodesigningResults (infos, task.OutputCodesignItems, platform);
 			} finally {
@@ -532,7 +532,7 @@ void VerifyCodesigningResults (CodesignInfo [] infos, ITaskItem [] outputCodesig
 					var metadata = item.GetMetadata (kvp.Key);
 					if (metadata == string.Empty && kvp.Value != string.Empty) {
 						failures.Add ($"Item '{info.ItemSpec}': Expected metadata '{kvp.Key}' not found (with value '{kvp.Value}').");
-					} else if (!string.Equals (metadata, kvp.Value, StringComparison.Ordinal)) {
+					} else if (!string.Equals (metadata, kvp.Value)) {
 						failures.Add ($"Item '{info.ItemSpec}': Expected value '{kvp.Value}' for metadata '{kvp.Key}', but got '{metadata}' instead.\nExpected: {kvp.Value}\nActual:   {metadata}");
 					}
 				}
@@ -587,7 +587,7 @@ void Touch (string root, params string [] files)
 				if (file.EndsWith (".appex", StringComparison.OrdinalIgnoreCase) || file.EndsWith (".app", StringComparison.OrdinalIgnoreCase)) {
 					Directory.CreateDirectory (f);
 				} else {
-					Directory.CreateDirectory (Path.GetDirectoryName (file)!);
+					Directory.CreateDirectory (Path.GetDirectoryName (file));
 					File.WriteAllText (file, string.Empty);
 				}
 			}
@@ -637,7 +637,7 @@ public static Dictionary<string, string> CopyCustomMetadata (this ITaskItem self
 		{
 			var rv = new Dictionary<string, string> ();
 			foreach (DictionaryEntry de in self.CloneCustomMetadata ()) {
-				rv [(string) de.Key!] = (string) de.Value!;
+				rv [(string) de.Key] = (string) de.Value;
 			}
 			return rv;
 		}

tests/msbuild/Xamarin.MacDev.Tasks.Tests/TaskTests/ResolveNativeReferencesTaskTest.cs

@@ -38,7 +38,7 @@ public void Xcode12_x (string targetFrameworkMoniker, bool isSimulator, string a
 			// some architecture changes recently, e.g.
 			// in Xcode 12.1+ watchOS does not have an i386 architecture anymore
 			// on Xcode 12.2+ you get arm64 for all (iOS, tvOS and watchOS) simulators
-			var path = Path.Combine (Path.GetDirectoryName (GetType ().Assembly.Location)!, "Resources", "xcf-xcode12.2.plist");
+			var path = Path.Combine (Path.GetDirectoryName (GetType ().Assembly.Location), "Resources", "xcf-xcode12.2.plist");
 			var plist = PDictionary.FromFile (path)!;
 			var result = ResolveNativeReferences.TryResolveXCFramework (log, plist, "N/A", targetFrameworkMoniker, isSimulator, architecture, out var frameworkPath);
 			Assert.AreEqual (result, !string.IsNullOrEmpty (expected), "result");
@@ -51,7 +51,7 @@ public void Xcode12_x (string targetFrameworkMoniker, bool isSimulator, string a
 		[TestCase (TargetFramework.Xamarin_WatchOS_1_0_String, true, "i386", "watchos-i386-simulator/XTest.framework/XTest")]
 		public void PreXcode12 (string targetFrameworkMoniker, bool isSimulator, string architecture, string expected)
 		{
-			var path = Path.Combine (Path.GetDirectoryName (GetType ().Assembly.Location)!, "Resources", "xcf-prexcode12.plist");
+			var path = Path.Combine (Path.GetDirectoryName (GetType ().Assembly.Location), "Resources", "xcf-prexcode12.plist");
 			var plist = PDictionary.FromFile (path)!;
 			var result = ResolveNativeReferences.TryResolveXCFramework (log, plist, "N/A", targetFrameworkMoniker, isSimulator, architecture, out var frameworkPath);
 			Assert.AreEqual (result, !string.IsNullOrEmpty (expected), "result");

tests/msbuild/Xamarin.MacDev.Tasks.Tests/Xamarin.MacDev.Tasks.Tests.csproj

@@ -1,7 +1,7 @@
 <?xml version="1.0" encoding="utf-8"?>
 <Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
-    <TargetFramework>net$(BundledNETCoreAppTargetFrameworkVersion)</TargetFramework>
+    <TargetFramework>net472</TargetFramework>
     <GenerateAssemblyInfo>false</GenerateAssemblyInfo>
     <DisableTransitiveProjectReferences>true</DisableTransitiveProjectReferences>
     <LangVersion>latest</LangVersion>
@@ -32,12 +32,12 @@
     <Reference Include="Xamarin.Localization.MSBuild" HintPath="..\..\..\msbuild\Xamarin.iOS.Tasks\bin\$(Configuration)\netstandard2.0\Xamarin.Localization.MSBuild.dll" />
     <PackageReference Include="System.Collections.Immutable" Version="8.0.0" />
     <PackageReference Include="NUnit" Version="3.12.0" />
-    <PackageReference Include="NUnit3TestAdapter" Version="3.15.1" />
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.4.1" />
-    <PackageReference Include="Microsoft.Build" Version="17.11.4" />
-    <PackageReference Include="Microsoft.Build.Framework" Version="17.11.4" />
-    <PackageReference Include="Microsoft.Build.Tasks.Core" Version="17.11.4" />
-    <PackageReference Include="Microsoft.Build.Utilities.Core" Version="17.11.4" />
+    <PackageReference Include="NUnit.ConsoleRunner" Version="3.11.1" />
+    <PackageReference Include="NUnit.Extension.NUnitV2ResultWriter" Version="3.6.0" />
+    <PackageReference Include="Microsoft.Build" Version="15.9.20" />
+    <PackageReference Include="Microsoft.Build.Framework" Version="15.9.20" />
+    <PackageReference Include="Microsoft.Build.Tasks.Core" Version="15.9.20" />
+    <PackageReference Include="Microsoft.Build.Utilities.Core" Version="15.9.20" />
   </ItemGroup>
 
   <Target Name="BuildTasksAssembly" AfterTargets="BeforeBuild">

tests/msbuild/Xamarin.MacDev.Tests/TestHelpers/BuildEngine.cs

@@ -246,7 +246,7 @@ static ExecutionResult MSBuild (ApplePlatform platform, string project, string t
 				foreach (var prop in properties)
 					args.Add ($"/p:{prop.Key}={prop.Value}");
 			}
-			var binlog = Path.Combine (Path.GetDirectoryName (project)!, $"log-{target}-{DateTime.Now:yyyyMMdd_HHmmss}.binlog");
+			var binlog = Path.Combine (Path.GetDirectoryName (project), $"log-{target}-{DateTime.Now:yyyyMMdd_HHmmss}.binlog");
 			args.Add ($"/bl:{binlog}");
 
 			var output = new StringBuilder ();

tests/msbuild/Xamarin.MacDev.Tests/Xamarin.MacDev.Tests.csproj

@@ -1,7 +1,7 @@
 <?xml version="1.0" encoding="utf-8"?>
 <Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
-    <TargetFramework>net$(BundledNETCoreAppTargetFrameworkVersion)</TargetFramework>
+    <TargetFramework>net472</TargetFramework>
     <GenerateAssemblyInfo>false</GenerateAssemblyInfo>
     <DisableTransitiveProjectReferences>true</DisableTransitiveProjectReferences>
     <LangVersion>latest</LangVersion>
@@ -11,8 +11,8 @@
   </PropertyGroup>
   <ItemGroup>
     <PackageReference Include="NUnit" Version="3.12.0" />
-    <PackageReference Include="NUnit3TestAdapter" Version="3.15.1" />
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.4.1" />
+    <PackageReference Include="NUnit.ConsoleRunner" Version="3.11.1" />
+    <PackageReference Include="NUnit.Extension.NUnitV2ResultWriter" Version="3.6.0" />
     <PackageReference Include="Mono.Cecil" Version="$(MonoCecilPackageVersion)" />
     <PackageReference Include="MSBuild.StructuredLogger" Version="$(MSBuildStructuredLoggerPackageVersion)" />
   </ItemGroup>

tests/xharness/Jenkins/NUnitTestTasksEnumerable.cs

@@ -19,24 +19,27 @@ public NUnitTestTasksEnumerable (Jenkins jenkins, IMlaunchProcessManager process
 
 		public IEnumerator<RunTestTask> GetEnumerator ()
 		{
-			var msbuildTasksTestsProject = new TestProject (TestLabel.Msbuild, Path.GetFullPath (Path.Combine (HarnessConfiguration.RootDirectory, "msbuild", "Xamarin.MacDev.Tasks.Tests", "Xamarin.MacDev.Tasks.Tests.csproj"))) {
+			var netstandard2Project = new TestProject (TestLabel.Msbuild, Path.GetFullPath (Path.Combine (HarnessConfiguration.RootDirectory, "msbuild", "Xamarin.MacDev.Tasks.Tests", "Xamarin.MacDev.Tasks.Tests.csproj"))) {
 				IsDotNetProject = true,
 			};
 			var env = new Dictionary<string, string>
 			{
 				{ "SYSTEM_MONO", this.jenkins.Harness.SYSTEM_MONO },
 			};
-			var buildiOSMSBuild = new MSBuildTask (jenkins: jenkins, testProject: msbuildTasksTestsProject, processManager: processManager) {
+			var buildiOSMSBuild = new MSBuildTask (jenkins: jenkins, testProject: netstandard2Project, processManager: processManager) {
 				SpecifyPlatform = false,
+				SpecifyConfiguration = true,
 				ProjectConfiguration = "Debug",
-				Platform = TestPlatform.All,
-				Ignored = !jenkins.TestSelection.IsEnabled (TestLabel.Msbuild),
+				Platform = TestPlatform.iOS,
+				SolutionPath = Path.GetFullPath (Path.Combine (HarnessConfiguration.RootDirectory, "..", "msbuild", "Xamarin.MacDev.Tasks.sln")),
 				SupportsParallelExecution = false,
+				Environment = env,
 			};
-			var nunitExecutioniOSMSBuild = new DotNetTestTask (jenkins, buildiOSMSBuild, processManager) {
-				TestProject = msbuildTasksTestsProject,
+			var nunitExecutioniOSMSBuild = new NUnitExecuteTask (jenkins, buildiOSMSBuild, processManager) {
+				TestLibrary = Path.Combine (HarnessConfiguration.RootDirectory, "msbuild", "Xamarin.MacDev.Tasks.Tests", "bin", "Debug", "net472", "Xamarin.MacDev.Tasks.Tests.dll"),
+				TestProject = netstandard2Project,
 				ProjectConfiguration = "Debug",
-				Platform = TestPlatform.All,
+				Platform = TestPlatform.iOS,
 				TestName = "MSBuild tests",
 				Mode = "Tasks",
 				Timeout = TimeSpan.FromMinutes (60),
@@ -50,15 +53,18 @@ public IEnumerator<RunTestTask> GetEnumerator ()
 			};
 			var buildiOSMSBuildIntegration = new MSBuildTask (jenkins: jenkins, testProject: msbuildIntegrationTestsProject, processManager: processManager) {
 				SpecifyPlatform = false,
+				SpecifyConfiguration = true,
 				ProjectConfiguration = "Debug",
-				Platform = TestPlatform.All,
-				Ignored = !jenkins.TestSelection.IsEnabled (TestLabel.Msbuild),
+				Platform = TestPlatform.iOS,
+				SolutionPath = Path.GetFullPath (Path.Combine (HarnessConfiguration.RootDirectory, "..", "msbuild", "Xamarin.MacDev.Tasks.sln")),
 				SupportsParallelExecution = false,
+				Environment = env,
 			};
-			var nunitExecutioniOSMSBuildIntegration = new DotNetTestTask (jenkins, buildiOSMSBuildIntegration, processManager) {
+			var nunitExecutioniOSMSBuildIntegration = new NUnitExecuteTask (jenkins, buildiOSMSBuildIntegration, processManager) {
+				TestLibrary = Path.Combine (HarnessConfiguration.RootDirectory, "msbuild", "Xamarin.MacDev.Tests", "bin", "Debug", "net472", "Xamarin.MacDev.Tests.dll"),
 				TestProject = msbuildIntegrationTestsProject,
 				ProjectConfiguration = "Debug",
-				Platform = TestPlatform.All,
+				Platform = TestPlatform.iOS,
 				TestName = "MSBuild tests",
 				Mode = "Integration",
 				Timeout = TimeSpan.FromMinutes (120),