From 4ebd9babbfbd1ec75b6b309aa6f59c43c0b0a227 Mon Sep 17 00:00:00 2001 From: "Brett V. Forsgren" Date: Mon, 9 Dec 2024 17:22:19 -0700 Subject: [PATCH] install .NET SDKs as specified by repo's `global.json` files --- .../EntryPointTests.Update.cs | 2 +- .../Discover/SdkProjectDiscoveryTests.cs | 2 +- .../MockNuGetPackage.cs | 3 +- .../Utilities/MSBuildHelperTests.cs | 306 ++++++++++++++---- .../Analyze/AnalyzeWorker.cs | 3 + .../Analyze/DependencyFinder.cs | 2 + .../Discover/DiscoveryWorker.cs | 4 +- .../Discover/PackagesConfigDiscovery.cs | 4 +- .../Discover/SdkProjectDiscovery.cs | 30 +- .../NuGetUpdater.Core/ExperimentsManager.cs | 3 + .../Updater/LockFileUpdater.cs | 5 +- .../Updater/PackageReferenceUpdater.cs | 61 ++-- .../Updater/UpdaterWorker.cs | 2 +- .../Utilities/MSBuildHelper.cs | 54 +++- .../Utilities/NuGetHelper.cs | 4 +- .../Utilities/ProcessExtensions.cs | 52 ++- nuget/lib/dependabot/nuget/file_fetcher.rb | 1 + nuget/lib/dependabot/nuget/file_parser.rb | 1 + nuget/lib/dependabot/nuget/native_helpers.rb | 21 ++ nuget/updater/common.ps1 | 44 +++ nuget/updater/install-sdks.ps1 | 18 ++ nuget/updater/main.ps1 | 45 +-- 22 files changed, 506 insertions(+), 161 deletions(-) create mode 100755 nuget/updater/install-sdks.ps1 diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Cli.Test/EntryPointTests.Update.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Cli.Test/EntryPointTests.Update.cs index b11a681b87..6981b2d369 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Cli.Test/EntryPointTests.Update.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Cli.Test/EntryPointTests.Update.cs @@ -382,7 +382,7 @@ await File.WriteAllTextAsync(projectPath, """ workingDirectory = Path.Join(workingDirectory, workingDirectoryPath); } - var (exitCode, output, error) = await ProcessEx.RunAsync("dotnet", executableArgs, workingDirectory: workingDirectory); + var (exitCode, output, error) = await ProcessEx.RunDotnetWithoutMSBuildEnvironmentVariablesAsync(executableArgs, workingDirectory, new ExperimentsManager() { InstallDotnetSdks = false }); Assert.True(exitCode == 0, $"Error running update on unsupported SDK.\nSTDOUT:\n{output}\nSTDERR:\n{error}"); // verify project update diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Discover/SdkProjectDiscoveryTests.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Discover/SdkProjectDiscoveryTests.cs index 37b3adefa0..90f03bed81 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Discover/SdkProjectDiscoveryTests.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Discover/SdkProjectDiscoveryTests.cs @@ -488,7 +488,7 @@ private static async Task TestDiscoverAsync(string startingDirectory, string pro var logger = new TestLogger(); var fullProjectPath = Path.Combine(testDirectory.DirectoryPath, projectPath); var experimentsManager = new ExperimentsManager() { UseDirectDiscovery = true }; // the following method is direct discovery; this just makes the call to Validate... happy - var projectDiscovery = await SdkProjectDiscovery.DiscoverWithBinLogAsync(testDirectory.DirectoryPath, Path.GetDirectoryName(fullProjectPath)!, fullProjectPath, logger); + var projectDiscovery = await SdkProjectDiscovery.DiscoverWithBinLogAsync(testDirectory.DirectoryPath, Path.GetDirectoryName(fullProjectPath)!, fullProjectPath, experimentsManager, logger); ValidateProjectResults(expectedProjects, projectDiscovery, experimentsManager); } } diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/MockNuGetPackage.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/MockNuGetPackage.cs index 4a264efe4c..eae2e1237f 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/MockNuGetPackage.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/MockNuGetPackage.cs @@ -318,7 +318,8 @@ private static byte[] CreateAssembly(string assemblyName, string assemblyVersion """ ); - var (exitCode, stdout, stderr) = ProcessEx.RunAsync("dotnet", ["msbuild", projectPath, "/t:_ReportCurrentSdkVersion"]).Result; + var experimentsManager = new ExperimentsManager(); + var (exitCode, stdout, stderr) = ProcessEx.RunDotnetWithoutMSBuildEnvironmentVariablesAsync(["msbuild", projectPath, "/t:_ReportCurrentSdkVersion"], projectDir.FullName, experimentsManager).Result; if (exitCode != 0) { throw new Exception($"Failed to report the current SDK version:\n{stdout}\n{stderr}"); diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Utilities/MSBuildHelperTests.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Utilities/MSBuildHelperTests.cs index be262b70fc..5690e4ba1c 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Utilities/MSBuildHelperTests.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Utilities/MSBuildHelperTests.cs @@ -147,8 +147,10 @@ public async Task TopLevelPackageDependenciesCanBeDetermined(TestFile[] buildFil AssertEx.Equal(expectedTopLevelDependencies, actualTopLevelDependencies); } - [Fact] - public async Task AllPackageDependenciesCanBeTraversed() + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task AllPackageDependenciesCanBeTraversed(bool useExistingSdks) { using var temp = new TemporaryDirectory(); MockNuGetPackage[] testPackages = @@ -173,13 +175,16 @@ public async Task AllPackageDependenciesCanBeTraversed() temp.DirectoryPath, "netstandard2.0", [new Dependency("Package.A", "1.0.0", DependencyType.Unknown)], + new ExperimentsManager() { InstallDotnetSdks = useExistingSdks }, new TestLogger() ); AssertEx.Equal(expectedDependencies, actualDependencies); } - [Fact] - public async Task AllPackageDependencies_DoNotTruncateLongDependencyLists() + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task AllPackageDependencies_DoNotTruncateLongDependencyLists(bool useExistingSdks) { using var temp = new TemporaryDirectory(); MockNuGetPackage[] testPackages = @@ -301,7 +306,14 @@ public async Task AllPackageDependencies_DoNotTruncateLongDependencyLists() new Dependency("Package.2A", "1.0.0", DependencyType.Unknown), new Dependency("Package.2R", "18.0.0", DependencyType.Unknown), }; - var actualDependencies = await MSBuildHelper.GetAllPackageDependenciesAsync(temp.DirectoryPath, temp.DirectoryPath, "net8.0", packages, new TestLogger()); + var actualDependencies = await MSBuildHelper.GetAllPackageDependenciesAsync( + temp.DirectoryPath, + temp.DirectoryPath, + "net8.0", + packages, + new ExperimentsManager() { InstallDotnetSdks = useExistingSdks }, + new TestLogger() + ); for (int i = 0; i < actualDependencies.Length; i++) { var ad = actualDependencies[i]; @@ -312,8 +324,10 @@ public async Task AllPackageDependencies_DoNotTruncateLongDependencyLists() AssertEx.Equal(expectedDependencies, actualDependencies); } - [Fact] - public async Task AllPackageDependencies_DoNotIncludeUpdateOnlyPackages() + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task AllPackageDependencies_DoNotIncludeUpdateOnlyPackages(bool useExistingSdks) { using var temp = new TemporaryDirectory(); MockNuGetPackage[] testPackages = @@ -335,12 +349,21 @@ public async Task AllPackageDependencies_DoNotIncludeUpdateOnlyPackages() new Dependency("Package.B", "2.0.0", DependencyType.Unknown, TargetFrameworks: ["net8.0"]), new Dependency("Package.C", "3.0.0", DependencyType.Unknown, IsUpdate: true) }; - var actualDependencies = await MSBuildHelper.GetAllPackageDependenciesAsync(temp.DirectoryPath, temp.DirectoryPath, "net8.0", packages, new TestLogger()); + var actualDependencies = await MSBuildHelper.GetAllPackageDependenciesAsync( + temp.DirectoryPath, + temp.DirectoryPath, + "net8.0", + packages, + new ExperimentsManager() { InstallDotnetSdks = useExistingSdks }, + new TestLogger() + ); AssertEx.Equal(expectedDependencies, actualDependencies); } - [Fact] - public async Task GetAllPackageDependencies_NugetConfigInvalid_DoesNotThrow() + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task GetAllPackageDependencies_NugetConfigInvalid_DoesNotThrow(bool useExistingSdks) { using var temp = new TemporaryDirectory(); @@ -362,12 +385,15 @@ await File.WriteAllTextAsync( temp.DirectoryPath, "net8.0", [new Dependency("Some.Package", "4.5.11", DependencyType.Unknown)], + new ExperimentsManager() { InstallDotnetSdks = useExistingSdks }, new TestLogger() ); } - [Fact] - public async Task LocalPackageSourcesAreHonored() + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task LocalPackageSourcesAreHonored(bool useExistingSdks) { using var temp = new TemporaryDirectory(); @@ -402,14 +428,17 @@ await File.WriteAllTextAsync(Path.Join(temp.DirectoryPath, "NuGet.Config"), """ temp.DirectoryPath, "net8.0", [new Dependency("Package.A", "1.0.0", DependencyType.Unknown)], + new ExperimentsManager() { InstallDotnetSdks = useExistingSdks }, new TestLogger() ); AssertEx.Equal(expectedDependencies, actualDependencies); } - [Fact] - public async Task DependencyConflictsCanBeResolvedWithBruteForce() + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task DependencyConflictsCanBeResolvedWithBruteForce(bool useExistingSdks) { var repoRoot = Directory.CreateTempSubdirectory($"test_{nameof(DependencyConflictsCanBeResolvedWithBruteForce)}_"); MockNuGetPackage[] testPackages = @@ -450,7 +479,14 @@ await File.WriteAllTextAsync(projectPath, """ { new Dependency("Some.Other.Package", "1.2.0", DependencyType.PackageReference), }; - var resolvedDependencies = await MSBuildHelper.ResolveDependencyConflictsWithBruteForce(repoRoot.FullName, projectPath, "net8.0", dependencies, new TestLogger()); + var resolvedDependencies = await MSBuildHelper.ResolveDependencyConflictsWithBruteForce( + repoRoot.FullName, + projectPath, + "net8.0", + dependencies, + new ExperimentsManager() { InstallDotnetSdks = useExistingSdks }, + new TestLogger() + ); Assert.NotNull(resolvedDependencies); Assert.Equal(2, resolvedDependencies.Length); Assert.Equal("Some.Package", resolvedDependencies[0].Name); @@ -505,8 +541,10 @@ public void UpdateWithWorkloadsTargetFrameworks() #region // Updating root package // CS-Script Code to 2.0.0 requires its dependency Microsoft.CodeAnalysis.CSharp.Scripting to be 3.6.0 and its transitive dependency Microsoft.CodeAnalysis.Common to be 3.6.0 - [Fact] - public async Task DependencyConflictsCanBeResolvedNewUpdatingTopLevelPackage() + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task DependencyConflictsCanBeResolvedNewUpdatingTopLevelPackage(bool useExistingSdks) { using var tempDirectory = new TemporaryDirectory(); var projectPath = Path.Join(tempDirectory.DirectoryPath, "project.csproj"); @@ -535,7 +573,15 @@ await File.WriteAllTextAsync(projectPath, """ new Dependency("CS-Script.Core", "2.0.0", DependencyType.PackageReference), }; - var resolvedDependencies = await MSBuildHelper.ResolveDependencyConflicts(tempDirectory.DirectoryPath, projectPath, "net8.0", dependencies, update, new TestLogger()); + var resolvedDependencies = await MSBuildHelper.ResolveDependencyConflicts( + tempDirectory.DirectoryPath, + projectPath, + "net8.0", + dependencies, + update, + new ExperimentsManager() { InstallDotnetSdks = useExistingSdks }, + new TestLogger() + ); Assert.NotNull(resolvedDependencies); Assert.Equal(3, resolvedDependencies.Length); Assert.Equal("CS-Script.Core", resolvedDependencies[0].Name); @@ -547,8 +593,10 @@ await File.WriteAllTextAsync(projectPath, """ } // Updating a dependency (Microsoft.Bcl.AsyncInterfaces) of the root package (Azure.Core) will require the root package to also update, but since the dependency is not in the existing list, we do not include it - [Fact] - public async Task DependencyConflictsCanBeResolvedNewUpdatingNonExistingDependency() + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task DependencyConflictsCanBeResolvedNewUpdatingNonExistingDependency(bool useExistingSdks) { using var tempDirectory = new TemporaryDirectory(); var projectPath = Path.Join(tempDirectory.DirectoryPath, "project.csproj"); @@ -572,7 +620,15 @@ await File.WriteAllTextAsync(projectPath, """ new Dependency("Microsoft.Bcl.AsyncInterfaces", "1.1.1", DependencyType.Unknown) }; - var resolvedDependencies = await MSBuildHelper.ResolveDependencyConflicts(tempDirectory.DirectoryPath, projectPath, "net8.0", dependencies, update, new TestLogger()); + var resolvedDependencies = await MSBuildHelper.ResolveDependencyConflicts( + tempDirectory.DirectoryPath, + projectPath, + "net8.0", + dependencies, + update, + new ExperimentsManager() { InstallDotnetSdks = useExistingSdks }, + new TestLogger() + ); Assert.NotNull(resolvedDependencies); Assert.Single(resolvedDependencies); Assert.Equal("Azure.Core", resolvedDependencies[0].Name); @@ -582,8 +638,10 @@ await File.WriteAllTextAsync(projectPath, """ // Adding a reference // Newtonsoft.Json needs to update to 13.0.1. Although Newtonsoft.Json.Bson can use the original version of 12.0.1, for security vulnerabilities and // because there is no later version of Newtonsoft.Json.Bson 1.0.2, Newtonsoft.Json would be added to the existing list to prevent resolution - [Fact] - public async Task DependencyConflictsCanBeResolvedNewUpdatingNonExistentDependencyAndKeepingReference() + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task DependencyConflictsCanBeResolvedNewUpdatingNonExistentDependencyAndKeepingReference(bool useExistingSdks) { using var tempDirectory = new TemporaryDirectory(); var projectPath = Path.Join(tempDirectory.DirectoryPath, "project.csproj"); @@ -607,7 +665,15 @@ await File.WriteAllTextAsync(projectPath, """ new Dependency("Newtonsoft.Json", "13.0.1", DependencyType.Unknown) }; - var resolvedDependencies = await MSBuildHelper.ResolveDependencyConflicts(tempDirectory.DirectoryPath, projectPath, "net8.0", dependencies, update, new TestLogger()); + var resolvedDependencies = await MSBuildHelper.ResolveDependencyConflicts( + tempDirectory.DirectoryPath, + projectPath, + "net8.0", + dependencies, + update, + new ExperimentsManager() { InstallDotnetSdks = useExistingSdks }, + new TestLogger() + ); Assert.NotNull(resolvedDependencies); Assert.Equal(2, resolvedDependencies.Length); Assert.Equal("Newtonsoft.Json.Bson", resolvedDependencies[0].Name); @@ -620,8 +686,10 @@ await File.WriteAllTextAsync(projectPath, """ // Root package (Microsoft.CodeAnalysis.Compilers) and its dependencies (Microsoft.CodeAnalysis.CSharp), (Microsoft.CodeAnalysis.VisualBasic) are all 4.9.2 // These packages all require the transitive dependency of the root package (Microsoft.CodeAnalysis.Common) to be 4.9.2, but it's not in the existing list // If Microsoft.CodeAnalysis.Common is updated to 4.10.0, everything else updates and Microsoft.CoseAnalysis.Common is not kept in the existing list - [Fact] - public async Task DependencyConflictsCanBeResolvedNewTransitiveDependencyNotExisting() + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task DependencyConflictsCanBeResolvedNewTransitiveDependencyNotExisting(bool useExistingSdks) { using var tempDirectory = new TemporaryDirectory(); var projectPath = Path.Join(tempDirectory.DirectoryPath, "project.csproj"); @@ -649,7 +717,15 @@ await File.WriteAllTextAsync(projectPath, """ new Dependency("Microsoft.CodeAnalysis.Common", "4.10.0", DependencyType.PackageReference) }; - var resolvedDependencies = await MSBuildHelper.ResolveDependencyConflicts(tempDirectory.DirectoryPath, projectPath, "net8.0", dependencies, update, new TestLogger()); + var resolvedDependencies = await MSBuildHelper.ResolveDependencyConflicts( + tempDirectory.DirectoryPath, + projectPath, + "net8.0", + dependencies, + update, + new ExperimentsManager() { InstallDotnetSdks = useExistingSdks }, + new TestLogger() + ); Assert.NotNull(resolvedDependencies); Assert.Equal(3, resolvedDependencies.Length); Assert.Equal("Microsoft.CodeAnalysis.Compilers", resolvedDependencies[0].Name); @@ -662,8 +738,10 @@ await File.WriteAllTextAsync(projectPath, """ // Updating referenced dependency // The same as previous test, but the transitive dependency (Microsoft.CodeAnalysis.Common) is in the existing list - [Fact] - public async Task DependencyConflictsCanBeResolvedNewSingleTransitiveDependencyExisting() + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task DependencyConflictsCanBeResolvedNewSingleTransitiveDependencyExisting(bool useExistingSdks) { using var tempDirectory = new TemporaryDirectory(); var projectPath = Path.Join(tempDirectory.DirectoryPath, "project.csproj"); @@ -693,7 +771,15 @@ await File.WriteAllTextAsync(projectPath, """ new Dependency("Microsoft.CodeAnalysis.Common", "4.10.0", DependencyType.PackageReference) }; - var resolvedDependencies = await MSBuildHelper.ResolveDependencyConflicts(tempDirectory.DirectoryPath, projectPath, "net8.0", dependencies, update, new TestLogger()); + var resolvedDependencies = await MSBuildHelper.ResolveDependencyConflicts( + tempDirectory.DirectoryPath, + projectPath, + "net8.0", + dependencies, + update, + new ExperimentsManager() { InstallDotnetSdks = useExistingSdks }, + new TestLogger() + ); Assert.NotNull(resolvedDependencies); Assert.Equal(4, resolvedDependencies.Length); Assert.Equal("Microsoft.CodeAnalysis.Compilers", resolvedDependencies[0].Name); @@ -709,8 +795,10 @@ await File.WriteAllTextAsync(projectPath, """ // A combination of the third and fourth test, to measure efficiency of updating separate families // Keeping a dependency that was not included in the original list (Newtonsoft.Json) // Not keeping a dependency that was not included in the original list (Microsoft.CodeAnalysis.Common) - [Fact] - public async Task DependencyConflictsCanBeResolvedNewSelectiveAdditionPackages() + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task DependencyConflictsCanBeResolvedNewSelectiveAdditionPackages(bool useExistingSdks) { using var tempDirectory = new TemporaryDirectory(); var projectPath = Path.Join(tempDirectory.DirectoryPath, "project.csproj"); @@ -741,7 +829,15 @@ await File.WriteAllTextAsync(projectPath, """ new Dependency("Newtonsoft.Json", "13.0.1", DependencyType.Unknown) }; - var resolvedDependencies = await MSBuildHelper.ResolveDependencyConflicts(tempDirectory.DirectoryPath, projectPath, "net8.0", dependencies, update, new TestLogger()); + var resolvedDependencies = await MSBuildHelper.ResolveDependencyConflicts( + tempDirectory.DirectoryPath, + projectPath, + "net8.0", + dependencies, + update, + new ExperimentsManager() { InstallDotnetSdks = useExistingSdks }, + new TestLogger() + ); Assert.NotNull(resolvedDependencies); Assert.Equal(5, resolvedDependencies.Length); Assert.Equal("Microsoft.CodeAnalysis.Compilers", resolvedDependencies[0].Name); @@ -761,8 +857,10 @@ await File.WriteAllTextAsync(projectPath, """ // First family: Buildalyzer 7.0.1 requires Microsoft.CodeAnalysis.CSharp to be >= 4.0.0 and Microsoft.CodeAnalysis.Common to be 4.0.0 (@ 6.0.4, Microsoft.CodeAnalysis.Common isn't a dependency of buildalyzer) // Second family: Microsoft.CodeAnalysis.CSharp.Scripting 4.0.0 requires Microsoft.CodeAnalysis.CSharp 4.0.0 and Microsoft.CodeAnalysis.Common to be 4.0.0 (Specific version) // Updating Buildalyzer to 7.0.1 will update its transitive dependency (Microsoft.CodeAnalysis.Common) and then its transitive dependency's "family" - [Fact] - public async Task DependencyConflictsCanBeResolvedNewSharingDependency() + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task DependencyConflictsCanBeResolvedNewSharingDependency(bool useExistingSdks) { using var tempDirectory = new TemporaryDirectory(); var projectPath = Path.Join(tempDirectory.DirectoryPath, "project.csproj"); @@ -792,7 +890,15 @@ await File.WriteAllTextAsync(projectPath, """ new Dependency("Buildalyzer", "7.0.1", DependencyType.PackageReference), }; - var resolvedDependencies = await MSBuildHelper.ResolveDependencyConflicts(tempDirectory.DirectoryPath, projectPath, "net8.0", dependencies, update, new TestLogger()); + var resolvedDependencies = await MSBuildHelper.ResolveDependencyConflicts( + tempDirectory.DirectoryPath, + projectPath, + "net8.0", + dependencies, + update, + new ExperimentsManager() { InstallDotnetSdks = useExistingSdks }, + new TestLogger() + ); Assert.NotNull(resolvedDependencies); Assert.Equal(4, resolvedDependencies.Length); Assert.Equal("Buildalyzer", resolvedDependencies[0].Name); @@ -808,8 +914,10 @@ await File.WriteAllTextAsync(projectPath, """ // Updating two families at once to test efficiency // First family: Direct dependency (Microsoft.CodeAnalysis.Common) needs to be updated, which will then need to update in the existing list its dependency (System.Collections.Immutable) and "parent" (Microsoft.CodeAnalysis.Csharp.Scripting) // Second family: Updating the root package (Azure.Core) in the existing list will also need to update its dependency (Microsoft.Bcl.AsyncInterfaces) - [Fact] - public async Task DependencyConflictsCanBeResolvedNewUpdatingEntireFamily() + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task DependencyConflictsCanBeResolvedNewUpdatingEntireFamily(bool useExistingSdks) { using var tempDirectory = new TemporaryDirectory(); var projectPath = Path.Join(tempDirectory.DirectoryPath, "project.csproj"); @@ -841,7 +949,15 @@ await File.WriteAllTextAsync(projectPath, """ new Dependency("Azure.Core", "1.22.0", DependencyType.PackageReference) }; - var resolvedDependencies = await MSBuildHelper.ResolveDependencyConflicts(tempDirectory.DirectoryPath, projectPath, "net8.0", dependencies, update, new TestLogger()); + var resolvedDependencies = await MSBuildHelper.ResolveDependencyConflicts( + tempDirectory.DirectoryPath, + projectPath, + "net8.0", + dependencies, + update, + new ExperimentsManager() { InstallDotnetSdks = useExistingSdks }, + new TestLogger() + ); Assert.NotNull(resolvedDependencies); Assert.Equal(4, resolvedDependencies.Length); Assert.Equal("System.Collections.Immutable", resolvedDependencies[0].Name); @@ -855,8 +971,10 @@ await File.WriteAllTextAsync(projectPath, """ } // Similar to the last test, except Microsoft.CodeAnalysis.Common is in the existing list - [Fact] - public async Task DependencyConflictsCanBeResolvedNewUpdatingTopLevelAndDependency() + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task DependencyConflictsCanBeResolvedNewUpdatingTopLevelAndDependency(bool useExistingSdks) { using var tempDirectory = new TemporaryDirectory(); var projectPath = Path.Join(tempDirectory.DirectoryPath, "project.csproj"); @@ -890,7 +1008,15 @@ await File.WriteAllTextAsync(projectPath, """ new Dependency("Azure.Core", "1.22.0", DependencyType.PackageReference) }; - var resolvedDependencies = await MSBuildHelper.ResolveDependencyConflicts(tempDirectory.DirectoryPath, projectPath, "net8.0", dependencies, update, new TestLogger()); + var resolvedDependencies = await MSBuildHelper.ResolveDependencyConflicts( + tempDirectory.DirectoryPath, + projectPath, + "net8.0", + dependencies, + update, + new ExperimentsManager() { InstallDotnetSdks = useExistingSdks }, + new TestLogger() + ); Assert.NotNull(resolvedDependencies); Assert.Equal(5, resolvedDependencies.Length); Assert.Equal("System.Collections.Immutable", resolvedDependencies[0].Name); @@ -908,8 +1034,10 @@ await File.WriteAllTextAsync(projectPath, """ // Out of scope test: AutoMapper.Extensions.Microsoft.DependencyInjection's versions are not yet compatible // To update root package (AutoMapper.Collection) to 10.0.0, its dependency (AutoMapper) needs to update to 13.0.0. // However, there is no higher version of AutoMapper's other "parent" (AutoMapper.Extensions.Microsoft.DependencyInjection) that is compatible with the new version - [Fact] - public async Task DependencyConflictsCanBeResolvedNewOutOfScope() + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task DependencyConflictsCanBeResolvedNewOutOfScope(bool useExistingSdks) { using var tempDirectory = new TemporaryDirectory(); var projectPath = Path.Join(tempDirectory.DirectoryPath, "project.csproj"); @@ -937,7 +1065,15 @@ await File.WriteAllTextAsync(projectPath, """ new Dependency("AutoMapper.Collection", "10.0.0", DependencyType.PackageReference) }; - var resolvedDependencies = await MSBuildHelper.ResolveDependencyConflicts(tempDirectory.DirectoryPath, projectPath, "net8.0", dependencies, update, new TestLogger()); + var resolvedDependencies = await MSBuildHelper.ResolveDependencyConflicts( + tempDirectory.DirectoryPath, + projectPath, + "net8.0", + dependencies, + update, + new ExperimentsManager() { InstallDotnetSdks = useExistingSdks }, + new TestLogger() + ); Assert.NotNull(resolvedDependencies); Assert.Equal(3, resolvedDependencies.Length); Assert.Equal("AutoMapper.Extensions.Microsoft.DependencyInjection", resolvedDependencies[0].Name); @@ -949,8 +1085,10 @@ await File.WriteAllTextAsync(projectPath, """ } // Two dependencies (Microsoft.Extensions.Caching.Memory), (Microsoft.EntityFrameworkCore.Analyzers) used by the same parent (Microsoft.EntityFrameworkCore), updating one of the dependencies - [Fact] - public async Task DependencyConflictsCanBeResolvedNewTwoDependenciesShareSameParent() + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task DependencyConflictsCanBeResolvedNewTwoDependenciesShareSameParent(bool useExistingSdks) { using var tempDirectory = new TemporaryDirectory(); var projectPath = Path.Join(tempDirectory.DirectoryPath, "project.csproj"); @@ -976,7 +1114,15 @@ await File.WriteAllTextAsync(projectPath, """ new Dependency("Microsoft.Extensions.Caching.Memory", "8.0.0", DependencyType.PackageReference) }; - var resolvedDependencies = await MSBuildHelper.ResolveDependencyConflicts(tempDirectory.DirectoryPath, projectPath, "net8.0", dependencies, update, new TestLogger()); + var resolvedDependencies = await MSBuildHelper.ResolveDependencyConflicts( + tempDirectory.DirectoryPath, + projectPath, + "net8.0", + dependencies, + update, + new ExperimentsManager() { InstallDotnetSdks = useExistingSdks }, + new TestLogger() + ); Assert.NotNull(resolvedDependencies); Assert.Equal(2, resolvedDependencies.Length); Assert.Equal("Microsoft.EntityFrameworkCore", resolvedDependencies[0].Name); @@ -987,8 +1133,10 @@ await File.WriteAllTextAsync(projectPath, """ // Updating referenced package // 4 dependency chain to be updated. Since the package to be updated is in the existing list, do not update its parents since we want to change as little as possible - [Fact] - public async Task DependencyConflictsCanBeResolvedNewFamilyOfFourExisting() + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task DependencyConflictsCanBeResolvedNewFamilyOfFourExisting(bool useExistingSdks) { using var tempDirectory = new TemporaryDirectory(); var projectPath = Path.Join(tempDirectory.DirectoryPath, "project.csproj"); @@ -1018,7 +1166,15 @@ await File.WriteAllTextAsync(projectPath, """ new Dependency("Microsoft.EntityFrameworkCore.Analyzers", "8.0.0", DependencyType.PackageReference) }; - var resolvedDependencies = await MSBuildHelper.ResolveDependencyConflicts(tempDirectory.DirectoryPath, projectPath, "net8.0", dependencies, update, new TestLogger()); + var resolvedDependencies = await MSBuildHelper.ResolveDependencyConflicts( + tempDirectory.DirectoryPath, + projectPath, + "net8.0", + dependencies, + update, + new ExperimentsManager() { InstallDotnetSdks = useExistingSdks }, + new TestLogger() + ); Assert.NotNull(resolvedDependencies); Assert.Equal(4, resolvedDependencies.Length); Assert.Equal("Microsoft.EntityFrameworkCore.Design", resolvedDependencies[0].Name); @@ -1033,8 +1189,10 @@ await File.WriteAllTextAsync(projectPath, """ // Updating unreferenced package // 4 dependency chain to be updated, dependency to be updated is not in the existing list, so its family will all be updated - [Fact] - public async Task DependencyConflictsCanBeResolvedNewFamilyOfFourNotInExisting() + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task DependencyConflictsCanBeResolvedNewFamilyOfFourNotInExisting(bool useExistingSdks) { using var tempDirectory = new TemporaryDirectory(); var projectPath = Path.Join(tempDirectory.DirectoryPath, "project.csproj"); @@ -1062,7 +1220,15 @@ await File.WriteAllTextAsync(projectPath, """ new Dependency("Microsoft.EntityFrameworkCore.Analyzers", "8.0.0", DependencyType.PackageReference) }; - var resolvedDependencies = await MSBuildHelper.ResolveDependencyConflicts(tempDirectory.DirectoryPath, projectPath, "net8.0", dependencies, update, new TestLogger()); + var resolvedDependencies = await MSBuildHelper.ResolveDependencyConflicts( + tempDirectory.DirectoryPath, + projectPath, + "net8.0", + dependencies, + update, + new ExperimentsManager() { InstallDotnetSdks = useExistingSdks }, + new TestLogger() + ); Assert.NotNull(resolvedDependencies); Assert.Equal(3, resolvedDependencies.Length); Assert.Equal("Microsoft.EntityFrameworkCore.Design", resolvedDependencies[0].Name); @@ -1075,8 +1241,10 @@ await File.WriteAllTextAsync(projectPath, """ // Updating a referenced transitive dependency // Updating a transtitive dependency (System.Collections.Immutable) to 8.0.0, which will update its "parent" (Microsoft.CodeAnalysis.CSharp) and its "grandparent" (Microsoft.CodeAnalysis.CSharp.Workspaces) to update - [Fact] - public async Task DependencyConflictsCanBeResolvedNewFamilyOfFourSpecificExisting() + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task DependencyConflictsCanBeResolvedNewFamilyOfFourSpecificExisting(bool useExistingSdks) { using var tempDirectory = new TemporaryDirectory(); var projectPath = Path.Join(tempDirectory.DirectoryPath, "project.csproj"); @@ -1106,7 +1274,15 @@ await File.WriteAllTextAsync(projectPath, """ new Dependency("System.Collections.Immutable", "8.0.0", DependencyType.PackageReference), }; - var resolvedDependencies = await MSBuildHelper.ResolveDependencyConflicts(tempDirectory.DirectoryPath, projectPath, "net8.0", dependencies, update, new TestLogger()); + var resolvedDependencies = await MSBuildHelper.ResolveDependencyConflicts( + tempDirectory.DirectoryPath, + projectPath, + "net8.0", + dependencies, + update, + new ExperimentsManager() { InstallDotnetSdks = useExistingSdks }, + new TestLogger() + ); Assert.NotNull(resolvedDependencies); Assert.Equal(4, resolvedDependencies.Length); Assert.Equal("System.Collections.Immutable", resolvedDependencies[0].Name); @@ -1120,8 +1296,10 @@ await File.WriteAllTextAsync(projectPath, """ } // Similar to the last test, with the "grandchild" (System.Collections.Immutable) not in the existing list - [Fact] - public async Task DependencyConflictsCanBeResolvedNewFamilyOfFourSpecificNotInExisting() + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task DependencyConflictsCanBeResolvedNewFamilyOfFourSpecificNotInExisting(bool useExistingSdks) { using var tempDirectory = new TemporaryDirectory(); var projectPath = Path.Join(tempDirectory.DirectoryPath, "project.csproj"); @@ -1150,7 +1328,15 @@ await File.WriteAllTextAsync(projectPath, """ new Dependency("System.Collections.Immutable", "8.0.0", DependencyType.PackageReference), }; - var resolvedDependencies = await MSBuildHelper.ResolveDependencyConflicts(tempDirectory.DirectoryPath, projectPath, "net8.0", dependencies, update, new TestLogger()); + var resolvedDependencies = await MSBuildHelper.ResolveDependencyConflicts( + tempDirectory.DirectoryPath, + projectPath, + "net8.0", + dependencies, + update, + new ExperimentsManager() { InstallDotnetSdks = useExistingSdks }, + new TestLogger() + ); Assert.NotNull(resolvedDependencies); Assert.Equal(3, resolvedDependencies.Length); Assert.Equal("Microsoft.CodeAnalysis.CSharp.Workspaces", resolvedDependencies[0].Name); diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Analyze/AnalyzeWorker.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Analyze/AnalyzeWorker.cs index 99207ca5d5..3918567fca 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Analyze/AnalyzeWorker.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Analyze/AnalyzeWorker.cs @@ -142,6 +142,7 @@ public async Task RunAsync(string repoRoot, WorkspaceDiscoveryRe dependenciesToUpdate, updatedVersion, nugetContext, + _experimentsManager, _logger, CancellationToken.None); } @@ -393,6 +394,7 @@ internal static async Task> FindUpdatedDependenciesAs ImmutableHashSet packageIds, NuGetVersion updatedVersion, NuGetContext nugetContext, + ExperimentsManager experimentsManager, ILogger logger, CancellationToken cancellationToken) { @@ -432,6 +434,7 @@ internal static async Task> FindUpdatedDependenciesAs packageIds, updatedVersion, nugetContext, + experimentsManager, logger, cancellationToken); diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Analyze/DependencyFinder.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Analyze/DependencyFinder.cs index 4581120ea9..b47988f724 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Analyze/DependencyFinder.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Analyze/DependencyFinder.cs @@ -14,6 +14,7 @@ public static async Task packageIds, NuGetVersion version, NuGetContext nugetContext, + ExperimentsManager experimentsManager, ILogger logger, CancellationToken cancellationToken) { @@ -30,6 +31,7 @@ public static async Task(); foreach (var dependency in dependencies) diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Discover/DiscoveryWorker.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Discover/DiscoveryWorker.cs index 9e58610f15..5480644883 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Discover/DiscoveryWorker.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Discover/DiscoveryWorker.cs @@ -137,7 +137,7 @@ private async Task TryRestoreMSBuildSdksAsync(string repoRootPath, string _logger.Info($" Restoring MSBuild SDKs: {string.Join(", ", keys)}"); - return await NuGetHelper.DownloadNuGetPackagesAsync(repoRootPath, workspacePath, msbuildSdks, logger); + return await NuGetHelper.DownloadNuGetPackagesAsync(repoRootPath, workspacePath, msbuildSdks, _experimentsManager, logger); } private async Task> RunForDirectoryAsnyc(string repoRootPath, string workspacePath) @@ -286,7 +286,7 @@ private async Task> RunForProjectPathsAsy _processedProjectPaths.Add(actualProjectPath); var relativeProjectPath = Path.GetRelativePath(workspacePath, actualProjectPath).NormalizePathToUnix(); - var packagesConfigResult = await PackagesConfigDiscovery.Discover(repoRootPath, workspacePath, actualProjectPath, _logger); + var packagesConfigResult = await PackagesConfigDiscovery.Discover(repoRootPath, workspacePath, actualProjectPath, _experimentsManager, _logger); var projectResults = await SdkProjectDiscovery.DiscoverAsync(repoRootPath, workspacePath, actualProjectPath, _experimentsManager, _logger); // Determine if there were unrestored MSBuildSdks diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Discover/PackagesConfigDiscovery.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Discover/PackagesConfigDiscovery.cs index d34bd7e6ea..86bd605346 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Discover/PackagesConfigDiscovery.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Discover/PackagesConfigDiscovery.cs @@ -6,7 +6,7 @@ namespace NuGetUpdater.Core.Discover; internal static class PackagesConfigDiscovery { - public static async Task Discover(string repoRootPath, string workspacePath, string projectPath, ILogger logger) + public static async Task Discover(string repoRootPath, string workspacePath, string projectPath, ExperimentsManager experimentsManager, ILogger logger) { var projectDirectory = Path.GetDirectoryName(projectPath)!; var additionalFiles = ProjectHelper.GetAllAdditionalFilesFromProject(projectPath, ProjectHelper.PathFormat.Full); @@ -27,7 +27,7 @@ internal static class PackagesConfigDiscovery .ToImmutableArray(); // generate `$(TargetFramework)` via MSBuild - var tfms = await MSBuildHelper.GetTargetFrameworkValuesFromProject(repoRootPath, projectPath, logger); + var tfms = await MSBuildHelper.GetTargetFrameworkValuesFromProject(repoRootPath, projectPath, experimentsManager, logger); var additionalFilesRelative = additionalFiles.Select(p => Path.GetRelativePath(projectDirectory, p).NormalizePathToUnix()).ToImmutableArray(); return new() diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Discover/SdkProjectDiscovery.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Discover/SdkProjectDiscovery.cs index d0b5a05ac2..266d746fa3 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Discover/SdkProjectDiscovery.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Discover/SdkProjectDiscovery.cs @@ -48,15 +48,15 @@ public static async Task> DiscoverAsync(s { if (experimentsManager.UseDirectDiscovery) { - return await DiscoverWithBinLogAsync(repoRootPath, workspacePath, startingProjectPath, logger); + return await DiscoverWithBinLogAsync(repoRootPath, workspacePath, startingProjectPath, experimentsManager, logger); } else { - return await DiscoverWithTempProjectAsync(repoRootPath, workspacePath, startingProjectPath, logger); + return await DiscoverWithTempProjectAsync(repoRootPath, workspacePath, startingProjectPath, experimentsManager, logger); } } - public static async Task> DiscoverWithBinLogAsync(string repoRootPath, string workspacePath, string startingProjectPath, ILogger logger) + public static async Task> DiscoverWithBinLogAsync(string repoRootPath, string workspacePath, string startingProjectPath, ExperimentsManager experimentsManager, ILogger logger) { // N.b., there are many paths used in this function. The MSBuild binary log always reports fully qualified paths, so that's what will be used // throughout until the very end when the appropriate kind of relative path is returned. @@ -84,7 +84,7 @@ public static async Task> DiscoverWithBin Dictionary> additionalFiles = new(PathComparer.Instance); // projectPath, additionalFiles - var tfms = await MSBuildHelper.GetTargetFrameworkValuesFromProject(repoRootPath, startingProjectPath, logger); + var tfms = await MSBuildHelper.GetTargetFrameworkValuesFromProject(repoRootPath, startingProjectPath, experimentsManager, logger); foreach (var tfm in tfms) { // create a binlog @@ -92,7 +92,7 @@ public static async Task> DiscoverWithBin try { // TODO: once the updater image has all relevant SDKs installed, we won't have to sideline global.json anymore - var (exitCode, stdOut, stdErr) = await MSBuildHelper.SidelineGlobalJsonAsync(startingProjectDirectory, repoRootPath, async () => + var (exitCode, stdOut, stdErr) = await MSBuildHelper.HandleGlobalJsonAsync(startingProjectDirectory, repoRootPath, experimentsManager, async () => { // the built-in target `GenerateBuildDependencyFile` forces resolution of all NuGet packages, but doesn't invoke a full build var dependencyDiscoveryTargetsPath = Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location)!, "DependencyDiscovery.targets"); @@ -102,10 +102,11 @@ public static async Task> DiscoverWithBin startingProjectPath, "/t:_DiscoverDependencies", $"/p:TargetFramework={tfm}", - $"/p:CustomAfterMicrosoftCommonCrossTargetingTargets={dependencyDiscoveryTargetsPath};CustomAfterMicrosoftCommonTargets={dependencyDiscoveryTargetsPath}", + $"/p:CustomAfterMicrosoftCommonCrossTargetingTargets={dependencyDiscoveryTargetsPath}", + $"/p:CustomAfterMicrosoftCommonTargets={dependencyDiscoveryTargetsPath}", $"/bl:{binLogPath}" }; - var (exitCode, stdOut, stdErr) = await ProcessEx.RunAsync("dotnet", args, workingDirectory: startingProjectDirectory); + var (exitCode, stdOut, stdErr) = await ProcessEx.RunDotnetWithoutMSBuildEnvironmentVariablesAsync(args, startingProjectDirectory, experimentsManager); return (exitCode, stdOut, stdErr); }, logger, retainMSBuildSdks: true); MSBuildHelper.ThrowOnUnauthenticatedFeed(stdOut); @@ -411,7 +412,7 @@ Dictionary> topLevelPackagesPerProject return property.Value; } - public static async Task> DiscoverWithTempProjectAsync(string repoRootPath, string workspacePath, string projectPath, ILogger logger) + public static async Task> DiscoverWithTempProjectAsync(string repoRootPath, string workspacePath, string projectPath, ExperimentsManager experimentsManager, ILogger logger) { // Determine which targets and props files contribute to the build. var (buildFiles, projectTargetFrameworks) = await MSBuildHelper.LoadBuildFilesAndTargetFrameworksAsync(repoRootPath, projectPath); @@ -476,7 +477,7 @@ public static async Task> DiscoverWithTem dependencies = dependencies .Select(d => d with { TargetFrameworks = tfms }) .ToImmutableArray(); - var transitiveDependencies = await GetTransitiveDependencies(repoRootPath, projectPath, tfms, dependencies, logger); + var transitiveDependencies = await GetTransitiveDependencies(repoRootPath, projectPath, tfms, dependencies, experimentsManager, logger); ImmutableArray allDependencies = dependencies.Concat(transitiveDependencies).Concat(sdkDependencies) .OrderBy(d => d.Name) .ToImmutableArray(); @@ -514,12 +515,19 @@ public static async Task> DiscoverWithTem return results.ToImmutable(); } - private static async Task> GetTransitiveDependencies(string repoRootPath, string projectPath, ImmutableArray tfms, ImmutableArray directDependencies, ILogger logger) + private static async Task> GetTransitiveDependencies( + string repoRootPath, + string projectPath, + ImmutableArray tfms, + ImmutableArray directDependencies, + ExperimentsManager experimentsManager, + ILogger logger + ) { Dictionary transitiveDependencies = new(StringComparer.OrdinalIgnoreCase); foreach (var tfm in tfms) { - var tfmDependencies = await MSBuildHelper.GetAllPackageDependenciesAsync(repoRootPath, projectPath, tfm, directDependencies, logger); + var tfmDependencies = await MSBuildHelper.GetAllPackageDependenciesAsync(repoRootPath, projectPath, tfm, directDependencies, experimentsManager, logger); foreach (var dependency in tfmDependencies.Where(d => d.IsTransitive)) { if (!transitiveDependencies.TryGetValue(dependency.Name, out var existingDependency)) diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/ExperimentsManager.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/ExperimentsManager.cs index ea42e96971..8e3cc01221 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/ExperimentsManager.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/ExperimentsManager.cs @@ -6,6 +6,7 @@ namespace NuGetUpdater.Core; public record ExperimentsManager { + public bool InstallDotnetSdks { get; init; } = false; public bool UseLegacyDependencySolver { get; init; } = false; public bool UseDirectDiscovery { get; init; } = false; @@ -13,6 +14,7 @@ public Dictionary ToDictionary() { return new() { + ["nuget_install_dotnet_sdks"] = InstallDotnetSdks, ["nuget_legacy_dependency_solver"] = UseLegacyDependencySolver, ["nuget_use_direct_discovery"] = UseDirectDiscovery, }; @@ -22,6 +24,7 @@ public static ExperimentsManager GetExperimentsManager(Dictionary + await MSBuildHelper.HandleGlobalJsonAsync(projectDirectory, repoRootPath, experimentsManager, async () => { - var (exitCode, stdout, stderr) = await ProcessEx.RunAsync("dotnet", ["restore", "--force-evaluate", projectPath], workingDirectory: projectDirectory); + var (exitCode, stdout, stderr) = await ProcessEx.RunDotnetWithoutMSBuildEnvironmentVariablesAsync(["restore", "--force-evaluate", projectPath], projectDirectory, experimentsManager); if (exitCode != 0) { logger.Error($" Lock file update failed.\nSTDOUT:\n{stdout}\nSTDERR:\n{stderr}"); diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Updater/PackageReferenceUpdater.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Updater/PackageReferenceUpdater.cs index 02734c0c42..da0a0f8e99 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Updater/PackageReferenceUpdater.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Updater/PackageReferenceUpdater.cs @@ -37,17 +37,17 @@ public static async Task UpdateDependencyAsync( // Get the set of all top-level dependencies in the current project var topLevelDependencies = MSBuildHelper.GetTopLevelPackageDependencyInfos(buildFiles).ToArray(); - if (!await DoesDependencyRequireUpdateAsync(repoRootPath, projectPath, tfms, topLevelDependencies, dependencyName, newDependencyVersion, logger)) + if (!await DoesDependencyRequireUpdateAsync(repoRootPath, projectPath, tfms, topLevelDependencies, dependencyName, newDependencyVersion, experimentsManager, logger)) { return; } - var peerDependencies = await GetUpdatedPeerDependenciesAsync(repoRootPath, projectPath, tfms, dependencyName, newDependencyVersion, logger); + var peerDependencies = await GetUpdatedPeerDependenciesAsync(repoRootPath, projectPath, tfms, dependencyName, newDependencyVersion, experimentsManager, logger); if (experimentsManager.UseLegacyDependencySolver) { if (isTransitive) { - await UpdateTransitiveDependencyAsync(repoRootPath, projectPath, dependencyName, newDependencyVersion, buildFiles, logger); + await UpdateTransitiveDependencyAsync(repoRootPath, projectPath, dependencyName, newDependencyVersion, buildFiles, experimentsManager, logger); } else { @@ -56,7 +56,7 @@ public static async Task UpdateDependencyAsync( return; } - await UpdateTopLevelDepdendency(repoRootPath, buildFiles, tfms, dependencyName, previousDependencyVersion, newDependencyVersion, peerDependencies, logger); + await UpdateTopLevelDepdendency(repoRootPath, buildFiles, tfms, dependencyName, previousDependencyVersion, newDependencyVersion, peerDependencies, experimentsManager, logger); } } else @@ -66,10 +66,10 @@ public static async Task UpdateDependencyAsync( return; } - await UpdateDependencyWithConflictResolution(repoRootPath, buildFiles, tfms, projectPath, dependencyName, previousDependencyVersion, newDependencyVersion, isTransitive, peerDependencies, logger); + await UpdateDependencyWithConflictResolution(repoRootPath, buildFiles, tfms, projectPath, dependencyName, previousDependencyVersion, newDependencyVersion, isTransitive, peerDependencies, experimentsManager, logger); } - if (!await AreDependenciesCoherentAsync(repoRootPath, projectPath, dependencyName, logger, buildFiles, tfms)) + if (!await AreDependenciesCoherentAsync(repoRootPath, projectPath, dependencyName, buildFiles, tfms, experimentsManager, logger)) { return; } @@ -87,6 +87,7 @@ public static async Task UpdateDependencyWithConflictResolution( string newDependencyVersion, bool isTransitive, IDictionary peerDependencies, + ExperimentsManager experimentsManager, ILogger logger) { var topLevelDependencies = MSBuildHelper.GetTopLevelPackageDependencyInfos(buildFiles).ToArray(); @@ -107,7 +108,7 @@ public static async Task UpdateDependencyWithConflictResolution( { foreach (var tfm in targetFrameworks) { - var resolvedDependencies = await MSBuildHelper.ResolveDependencyConflicts(repoRootPath, projectFile.Path, tfm, topLevelDependencies, dependenciesToUpdate, logger); + var resolvedDependencies = await MSBuildHelper.ResolveDependencyConflicts(repoRootPath, projectFile.Path, tfm, topLevelDependencies, dependenciesToUpdate, experimentsManager, logger); if (resolvedDependencies is null) { logger.Warn($" Unable to resolve dependency conflicts for {projectFile.Path}."); @@ -118,7 +119,7 @@ public static async Task UpdateDependencyWithConflictResolution( if (isTransitive && !isDependencyTopLevel && isDependencyInResolutionSet) { // a transitive dependency had to be pinned; add it here - await UpdateTransitiveDependencyAsync(repoRootPath, projectPath, dependencyName, newDependencyVersion, buildFiles, logger); + await UpdateTransitiveDependencyAsync(repoRootPath, projectPath, dependencyName, newDependencyVersion, buildFiles, experimentsManager, logger); } // update all resolved dependencies that aren't the initial dependency @@ -143,6 +144,7 @@ private static async Task DoesDependencyRequireUpdateAsync( Dependency[] topLevelDependencies, string dependencyName, string newDependencyVersion, + ExperimentsManager experimentsManager, ILogger logger) { var newDependencyNuGetVersion = NuGetVersion.Parse(newDependencyVersion); @@ -157,6 +159,7 @@ private static async Task DoesDependencyRequireUpdateAsync( projectPath, tfm, topLevelDependencies, + experimentsManager, logger); foreach (var dependency in dependencies) { @@ -203,7 +206,15 @@ private static async Task DoesDependencyRequireUpdateAsync( return true; } - private static async Task UpdateTransitiveDependencyAsync(string repoRootPath, string projectPath, string dependencyName, string newDependencyVersion, ImmutableArray buildFiles, ILogger logger) + private static async Task UpdateTransitiveDependencyAsync( + string repoRootPath, + string projectPath, + string dependencyName, + string newDependencyVersion, + ImmutableArray buildFiles, + ExperimentsManager experimentsManager, + ILogger logger + ) { var directoryPackagesWithPinning = buildFiles.OfType() .FirstOrDefault(bf => IsCpmTransitivePinningEnabled(bf)); @@ -213,7 +224,7 @@ private static async Task UpdateTransitiveDependencyAsync(string repoRootPath, s } else { - await AddTransitiveDependencyAsync(repoRootPath, projectPath, dependencyName, newDependencyVersion, logger); + await AddTransitiveDependencyAsync(repoRootPath, projectPath, dependencyName, newDependencyVersion, experimentsManager, logger); } } @@ -302,15 +313,19 @@ private static void PinTransitiveDependency(ProjectBuildFile directoryPackages, directoryPackages.Update(updatedXml); } - private static async Task AddTransitiveDependencyAsync(string repoRootPath, string projectPath, string dependencyName, string newDependencyVersion, ILogger logger) + private static async Task AddTransitiveDependencyAsync(string repoRootPath, string projectPath, string dependencyName, string newDependencyVersion, ExperimentsManager experimentsManager, ILogger logger) { var projectDirectory = Path.GetDirectoryName(projectPath)!; - await MSBuildHelper.SidelineGlobalJsonAsync(projectDirectory, repoRootPath, async () => + await MSBuildHelper.HandleGlobalJsonAsync(projectDirectory, repoRootPath, experimentsManager, async () => { logger.Info($" Adding [{dependencyName}/{newDependencyVersion}] as a top-level package reference."); // see https://learn.microsoft.com/nuget/consume-packages/install-use-packages-dotnet-cli - var (exitCode, stdout, stderr) = await ProcessEx.RunAsync("dotnet", ["add", projectPath, "package", dependencyName, "--version", newDependencyVersion], workingDirectory: projectDirectory); + var (exitCode, stdout, stderr) = await ProcessEx.RunDotnetWithoutMSBuildEnvironmentVariablesAsync( + ["add", projectPath, "package", dependencyName, "--version", newDependencyVersion], + projectDirectory, + experimentsManager + ); MSBuildHelper.ThrowOnUnauthenticatedFeed(stdout); if (exitCode != 0) { @@ -331,13 +346,14 @@ await MSBuildHelper.SidelineGlobalJsonAsync(projectDirectory, repoRootPath, asyn string[] tfms, string dependencyName, string newDependencyVersion, + ExperimentsManager experimentsManager, ILogger logger) { var newDependency = new[] { new Dependency(dependencyName, newDependencyVersion, DependencyType.Unknown) }; var tfmsAndDependencies = new Dictionary(); foreach (var tfm in tfms) { - var dependencies = await MSBuildHelper.GetAllPackageDependenciesAsync(repoRootPath, projectPath, tfm, newDependency, logger); + var dependencies = await MSBuildHelper.GetAllPackageDependenciesAsync(repoRootPath, projectPath, tfm, newDependency, experimentsManager, logger); tfmsAndDependencies[tfm] = dependencies; } @@ -386,6 +402,7 @@ private static async Task UpdateTopLevelDepdendency( string previousDependencyVersion, string newDependencyVersion, IDictionary peerDependencies, + ExperimentsManager experimentsManager, ILogger logger) { // update dependencies... @@ -407,7 +424,7 @@ private static async Task UpdateTopLevelDepdendency( { foreach (string tfm in targetFrameworks) { - var resolvedDependencies = await MSBuildHelper.ResolveDependencyConflictsWithBruteForce(repoRootPath, projectFile.Path, tfm, updatedTopLevelDependencies, logger); + var resolvedDependencies = await MSBuildHelper.ResolveDependencyConflictsWithBruteForce(repoRootPath, projectFile.Path, tfm, updatedTopLevelDependencies, experimentsManager, logger); if (resolvedDependencies is null) { logger.Info($" Unable to resolve dependency conflicts for {projectFile.Path}."); @@ -697,13 +714,21 @@ private static IEnumerable FindPackageNodes( ?? e.GetAttributeOrSubElementValue("VersionOverride", StringComparison.OrdinalIgnoreCase)) is not null; }); - private static async Task AreDependenciesCoherentAsync(string repoRootPath, string projectPath, string dependencyName, ILogger logger, ImmutableArray buildFiles, string[] tfms) + private static async Task AreDependenciesCoherentAsync( + string repoRootPath, + string projectPath, + string dependencyName, + ImmutableArray buildFiles, + string[] tfms, + ExperimentsManager experimentsManager, + ILogger logger + ) { var updatedTopLevelDependencies = MSBuildHelper.GetTopLevelPackageDependencyInfos(buildFiles).ToArray(); foreach (var tfm in tfms) { - var updatedPackages = await MSBuildHelper.GetAllPackageDependenciesAsync(repoRootPath, projectPath, tfm, updatedTopLevelDependencies, logger); - var dependenciesAreCoherent = await MSBuildHelper.DependenciesAreCoherentAsync(repoRootPath, projectPath, tfm, updatedPackages, logger); + var updatedPackages = await MSBuildHelper.GetAllPackageDependenciesAsync(repoRootPath, projectPath, tfm, updatedTopLevelDependencies, experimentsManager, logger); + var dependenciesAreCoherent = await MSBuildHelper.DependenciesAreCoherentAsync(repoRootPath, projectPath, tfm, updatedPackages, experimentsManager, logger); if (!dependenciesAreCoherent) { logger.Warn($" Package [{dependencyName}] could not be updated in [{projectPath}] because it would cause a dependency conflict."); diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Updater/UpdaterWorker.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Updater/UpdaterWorker.cs index 78d168c645..f6aaae756c 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Updater/UpdaterWorker.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Updater/UpdaterWorker.cs @@ -232,7 +232,7 @@ private async Task RunUpdaterAsync( var packagesLockFullPath = additionalFiles.Where(p => Path.GetFileName(p).Equals(ProjectHelper.PackagesLockJsonFileName, StringComparison.OrdinalIgnoreCase)).FirstOrDefault(); if (packagesLockFullPath is not null) { - await LockFileUpdater.UpdateLockFileAsync(repoRootPath, projectPath, _logger); + await LockFileUpdater.UpdateLockFileAsync(repoRootPath, projectPath, _experimentsManager, _logger); } } } diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Utilities/MSBuildHelper.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Utilities/MSBuildHelper.cs index dda3a7d7a1..32dc812dbf 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Utilities/MSBuildHelper.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Utilities/MSBuildHelper.cs @@ -34,7 +34,8 @@ public static void RegisterMSBuild(string currentDirectory, string rootDirectory // Ensure MSBuild types are registered before calling a method that loads the types if (!IsMSBuildRegistered) { - SidelineGlobalJsonAsync(currentDirectory, rootDirectory, () => + var experimentsManager = new ExperimentsManager() { InstallDotnetSdks = false }; // `global.json` definitely needs to be moved for this operation + HandleGlobalJsonAsync(currentDirectory, rootDirectory, experimentsManager, () => { var defaultInstance = MSBuildLocator.QueryVisualStudioInstances().First(); MSBuildPath = defaultInstance.MSBuildPath; @@ -44,9 +45,23 @@ public static void RegisterMSBuild(string currentDirectory, string rootDirectory } } - public static async Task SidelineGlobalJsonAsync(string currentDirectory, string rootDirectory, Func> action, ILogger? logger = null, bool retainMSBuildSdks = false) + public static async Task HandleGlobalJsonAsync( + string currentDirectory, + string rootDirectory, + ExperimentsManager experimentsManager, + Func> action, + ILogger? logger = null, + bool retainMSBuildSdks = false + ) { logger ??= new ConsoleLogger(); + if (experimentsManager.InstallDotnetSdks) + { + logger.Info($"{nameof(ExperimentsManager.InstallDotnetSdks)} == true; retaining `global.json` contents."); + var result = await action(); + return result; + } + var candidateDirectories = PathHelper.GetAllDirectoriesToRoot(currentDirectory, rootDirectory); var globalJsonPaths = candidateDirectories.Select(d => Path.Combine(d, "global.json")).Where(File.Exists).Select(p => (p, p + Guid.NewGuid().ToString())).ToArray(); foreach (var (globalJsonPath, tempGlobalJsonPath) in globalJsonPaths) @@ -322,13 +337,13 @@ public static bool TryGetPropertyName(string versionContent, [NotNullWhen(true)] return false; } - internal static async Task DependenciesAreCoherentAsync(string repoRoot, string projectPath, string targetFramework, Dependency[] packages, ILogger logger) + internal static async Task DependenciesAreCoherentAsync(string repoRoot, string projectPath, string targetFramework, Dependency[] packages, ExperimentsManager experimentsManager, ILogger logger) { var tempDirectory = Directory.CreateTempSubdirectory("package-dependency-coherence_"); try { var tempProjectPath = await CreateTempProjectAsync(tempDirectory, repoRoot, projectPath, targetFramework, packages, logger); - var (exitCode, stdOut, stdErr) = await ProcessEx.RunAsync("dotnet", ["restore", tempProjectPath], workingDirectory: tempDirectory.FullName); + var (exitCode, stdOut, stdErr) = await ProcessEx.RunDotnetWithoutMSBuildEnvironmentVariablesAsync(["restore", tempProjectPath], tempDirectory.FullName, experimentsManager); // NU1608: Detected package version outside of dependency constraint @@ -340,7 +355,7 @@ internal static async Task DependenciesAreCoherentAsync(string repoRoot, s } } - internal static async Task ResolveDependencyConflicts(string repoRoot, string projectPath, string targetFramework, Dependency[] packages, Dependency[] update, ILogger logger) + internal static async Task ResolveDependencyConflicts(string repoRoot, string projectPath, string targetFramework, Dependency[] packages, Dependency[] update, ExperimentsManager experimentsManager, ILogger logger) { var tempDirectory = Directory.CreateTempSubdirectory("package-dependency-coherence_"); PackageManager packageManager = new PackageManager(repoRoot, projectPath); @@ -348,7 +363,7 @@ internal static async Task DependenciesAreCoherentAsync(string repoRoot, s try { string tempProjectPath = await CreateTempProjectAsync(tempDirectory, repoRoot, projectPath, targetFramework, packages, logger); - var (exitCode, stdOut, stdErr) = await ProcessEx.RunAsync("dotnet", ["restore", tempProjectPath], workingDirectory: tempDirectory.FullName); + var (exitCode, stdOut, stdErr) = await ProcessEx.RunDotnetWithoutMSBuildEnvironmentVariablesAsync(["restore", tempProjectPath], tempDirectory.FullName, experimentsManager); // Add Dependency[] packages to List existingPackages List existingPackages = packages @@ -498,13 +513,13 @@ internal static async Task DependenciesAreCoherentAsync(string repoRoot, s } } - internal static async Task ResolveDependencyConflictsWithBruteForce(string repoRoot, string projectPath, string targetFramework, Dependency[] packages, ILogger logger) + internal static async Task ResolveDependencyConflictsWithBruteForce(string repoRoot, string projectPath, string targetFramework, Dependency[] packages, ExperimentsManager experimentsManager, ILogger logger) { var tempDirectory = Directory.CreateTempSubdirectory("package-dependency-coherence_"); try { var tempProjectPath = await CreateTempProjectAsync(tempDirectory, repoRoot, projectPath, targetFramework, packages, logger); - var (exitCode, stdOut, stdErr) = await ProcessEx.RunAsync("dotnet", ["restore", tempProjectPath], workingDirectory: tempDirectory.FullName); + var (exitCode, stdOut, stdErr) = await ProcessEx.RunDotnetWithoutMSBuildEnvironmentVariablesAsync(["restore", tempProjectPath], tempDirectory.FullName, experimentsManager); ThrowOnUnauthenticatedFeed(stdOut); // simple cases first @@ -529,6 +544,7 @@ internal static async Task DependenciesAreCoherentAsync(string repoRoot, s foreach ((string PackageName, NuGetVersion packageVersion) in badPackagesAndVersions) { // this command dumps a JSON object with all versions of the specified package from all package sources + // not using the `dotnet` execution method because we want to force the latest MSBuild and SDK to be used (exitCode, stdOut, stdErr) = await ProcessEx.RunAsync("dotnet", ["package", "search", PackageName, "--exact-match", "--format", "json"], workingDirectory: tempDirectory.FullName); if (exitCode != 0) { @@ -591,7 +607,7 @@ internal static async Task DependenciesAreCoherentAsync(string repoRoot, s return p; }).ToArray(); - if (await DependenciesAreCoherentAsync(repoRoot, projectPath, targetFramework, candidatePackages, logger)) + if (await DependenciesAreCoherentAsync(repoRoot, projectPath, targetFramework, candidatePackages, experimentsManager, logger)) { // return as soon as we find a coherent set return candidatePackages; @@ -749,14 +765,23 @@ await File.WriteAllTextAsync( return tempProjectPath; } - internal static async Task> GetTargetFrameworkValuesFromProject(string repoRoot, string projectPath, ILogger logger) + internal static async Task> GetTargetFrameworkValuesFromProject(string repoRoot, string projectPath, ExperimentsManager experimentsManager, ILogger logger) { - // TODO: once the updater image has all relevant SDKs installed, we won't have to sideline global.json anymore var projectDirectory = Path.GetDirectoryName(projectPath)!; - var (exitCode, stdOut, stdErr) = await SidelineGlobalJsonAsync(projectDirectory, repoRoot, async () => + var (exitCode, stdOut, stdErr) = await HandleGlobalJsonAsync(projectDirectory, repoRoot, experimentsManager, async () => { var targetsHelperPath = Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location)!, "TargetFrameworkReporter.targets"); - var (exitCode, stdOut, stdErr) = await ProcessEx.RunAsync("dotnet", ["build", projectPath, "/t:ReportTargetFramework", $"/p:CustomAfterMicrosoftCommonCrossTargetingTargets={targetsHelperPath};CustomAfterMicrosoftCommonTargets={targetsHelperPath}"], workingDirectory: Path.GetDirectoryName(projectPath)); + var (exitCode, stdOut, stdErr) = await ProcessEx.RunDotnetWithoutMSBuildEnvironmentVariablesAsync( + [ + "build", + projectPath, + "/t:ReportTargetFramework", + $"/p:CustomAfterMicrosoftCommonCrossTargetingTargets={targetsHelperPath}", + $"/p:CustomAfterMicrosoftCommonTargets={targetsHelperPath}" + ], + projectDirectory, + experimentsManager + ); return (exitCode, stdOut, stdErr); }); ThrowOnUnauthenticatedFeed(stdOut); @@ -806,6 +831,7 @@ internal static async Task GetAllPackageDependenciesAsync( string projectPath, string targetFramework, IReadOnlyCollection packages, + ExperimentsManager experimentsManager, ILogger logger) { var tempDirectory = Directory.CreateTempSubdirectory("package-dependency-resolution_"); @@ -814,7 +840,7 @@ internal static async Task GetAllPackageDependenciesAsync( var topLevelPackagesNames = packages.Select(p => p.Name).ToHashSet(StringComparer.OrdinalIgnoreCase); var tempProjectPath = await CreateTempProjectAsync(tempDirectory, repoRoot, projectPath, targetFramework, packages, logger); - var (exitCode, stdout, stderr) = await ProcessEx.RunAsync("dotnet", ["build", tempProjectPath, "/t:_ReportDependencies"], workingDirectory: tempDirectory.FullName); + var (exitCode, stdout, stderr) = await ProcessEx.RunDotnetWithoutMSBuildEnvironmentVariablesAsync(["build", tempProjectPath, "/t:_ReportDependencies"], tempDirectory.FullName, experimentsManager); ThrowOnUnauthenticatedFeed(stdout); if (exitCode == 0) diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Utilities/NuGetHelper.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Utilities/NuGetHelper.cs index 3a84fe8736..fb29c8e751 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Utilities/NuGetHelper.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Utilities/NuGetHelper.cs @@ -4,13 +4,13 @@ namespace NuGetUpdater.Core; internal static class NuGetHelper { - internal static async Task DownloadNuGetPackagesAsync(string repoRoot, string projectPath, IReadOnlyCollection packages, ILogger logger) + internal static async Task DownloadNuGetPackagesAsync(string repoRoot, string projectPath, IReadOnlyCollection packages, ExperimentsManager experimentsManager, ILogger logger) { var tempDirectory = Directory.CreateTempSubdirectory("msbuild_sdk_restore_"); try { var tempProjectPath = await MSBuildHelper.CreateTempProjectAsync(tempDirectory, repoRoot, projectPath, "netstandard2.0", packages, logger, usePackageDownload: true); - var (exitCode, stdOut, stdErr) = await ProcessEx.RunAsync("dotnet", ["restore", tempProjectPath]); + var (exitCode, stdOut, stdErr) = await ProcessEx.RunDotnetWithoutMSBuildEnvironmentVariablesAsync(["restore", tempProjectPath], tempDirectory.FullName, experimentsManager); return exitCode == 0; } diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Utilities/ProcessExtensions.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Utilities/ProcessExtensions.cs index aaace5eba7..2b2e5c07b7 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Utilities/ProcessExtensions.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Utilities/ProcessExtensions.cs @@ -5,19 +5,57 @@ namespace NuGetUpdater.Core; public static class ProcessEx { - public static Task<(int ExitCode, string Output, string Error)> RunAsync(string fileName, IEnumerable? arguments = null, string? workingDirectory = null) + /// + /// Run the `dotnet` command with the given values. This will exclude all `MSBuild*` environment variables from the execution. + /// + public static Task<(int ExitCode, string Output, string Error)> RunDotnetWithoutMSBuildEnvironmentVariablesAsync(IEnumerable arguments, string workingDirectory, ExperimentsManager experimentsManager) + { + var environmentVariablesToUnset = new List(); + if (experimentsManager.InstallDotnetSdks) + { + // If using the SDK specified by a `global.json` file, these environment variables need to be unset to + // allow the new process to discover the correct MSBuild binaries to load, and not load the ones that + // this process is using. + environmentVariablesToUnset.Add("MSBuildExtensionsPath"); + environmentVariablesToUnset.Add("MSBuildLoadMicrosoftTargetsReadOnly"); + environmentVariablesToUnset.Add("MSBUILDLOGIMPORTS"); + environmentVariablesToUnset.Add("MSBuildSDKsPath"); + environmentVariablesToUnset.Add("MSBUILDTARGETOUTPUTLOGGING"); + environmentVariablesToUnset.Add("MSBUILD_EXE_PATH"); + } + + var environmentVariableOverrides = environmentVariablesToUnset.Select(name => (name, (string?)null)); + return RunAsync("dotnet", + arguments, + workingDirectory, + environmentVariableOverrides + ); + } + + public static Task<(int ExitCode, string Output, string Error)> RunAsync( + string fileName, + IEnumerable? arguments = null, + string? workingDirectory = null, + IEnumerable<(string Name, string? Value)>? environmentVariableOverrides = null + ) { var tcs = new TaskCompletionSource<(int, string, string)>(); var redirectInitiated = new ManualResetEventSlim(); + var psi = new ProcessStartInfo(fileName, arguments ?? []) + { + UseShellExecute = false, // required to redirect output and set environment variables + RedirectStandardOutput = true, + RedirectStandardError = true, + }; + foreach (var (name, value) in environmentVariableOverrides ?? []) + { + psi.EnvironmentVariables[name] = value; + } + var process = new Process { - StartInfo = new ProcessStartInfo(fileName, arguments ?? []) - { - UseShellExecute = false, // required to redirect output - RedirectStandardOutput = true, - RedirectStandardError = true, - }, + StartInfo = psi, EnableRaisingEvents = true }; diff --git a/nuget/lib/dependabot/nuget/file_fetcher.rb b/nuget/lib/dependabot/nuget/file_fetcher.rb index 0b64718562..5888dc44c3 100644 --- a/nuget/lib/dependabot/nuget/file_fetcher.rb +++ b/nuget/lib/dependabot/nuget/file_fetcher.rb @@ -26,6 +26,7 @@ def self.required_files_message sig { override.returns(T::Array[DependencyFile]) } def fetch_files + NativeHelpers.install_dotnet_sdks discovery_json_reader = DiscoveryJsonReader.run_discovery_in_directory( repo_contents_path: T.must(repo_contents_path), directory: directory, diff --git a/nuget/lib/dependabot/nuget/file_parser.rb b/nuget/lib/dependabot/nuget/file_parser.rb index eac01a8ef6..b63c483a4a 100644 --- a/nuget/lib/dependabot/nuget/file_parser.rb +++ b/nuget/lib/dependabot/nuget/file_parser.rb @@ -57,6 +57,7 @@ def content_json sig { returns(T::Array[Dependabot::Dependency]) } def dependencies @dependencies ||= T.let(begin + NativeHelpers.install_dotnet_sdks directory = source&.directory || "/" discovery_json_reader = DiscoveryJsonReader.run_discovery_in_directory( repo_contents_path: T.must(repo_contents_path), diff --git a/nuget/lib/dependabot/nuget/native_helpers.rb b/nuget/lib/dependabot/nuget/native_helpers.rb index 887dc9b658..d6900de2f5 100644 --- a/nuget/lib/dependabot/nuget/native_helpers.rb +++ b/nuget/lib/dependabot/nuget/native_helpers.rb @@ -269,6 +269,27 @@ def self.run_nuget_updater_tool(job_path:, repo_root:, proj_path:, dependency:, end end + sig { void } + def self.install_dotnet_sdks + return unless Dependabot::Experiments.enabled?(:nuget_install_dotnet_sdks) + + # environment variables are required and the following will generate an actionable error message if they're not + _dependabot_job_path = ENV.fetch("DEPENDABOT_JOB_PATH") + _dependabot_repo_contents_path = ENV.fetch("DEPENDABOT_REPO_CONTENTS_PATH") + _dotnet_install_script_path = ENV.fetch("DOTNET_INSTALL_SCRIPT_PATH") + _dotnet_install_dir = ENV.fetch("DOTNET_INSTALL_DIR") + + # this environment variable is directly used + dependabot_home = ENV.fetch("DEPENDABOT_HOME") + + command = [ + "pwsh", + "#{dependabot_home}/dependabot-updater/bin/install-sdks.ps1" + ].join(" ") + output = SharedHelpers.run_shell_command(command) + puts output + end + sig { params(json: T::Hash[String, T.untyped]).void } def self.ensure_no_errors(json) error_type = T.let(json.fetch("ErrorType", nil), T.nilable(String)) diff --git a/nuget/updater/common.ps1 b/nuget/updater/common.ps1 index b72f11fb1f..657400d9d3 100755 --- a/nuget/updater/common.ps1 +++ b/nuget/updater/common.ps1 @@ -24,3 +24,47 @@ function Get-DirectoriesForSdkInstall([string] $repoRoot, [string[]]$updateDirec return ,$globalJsonPaths } + +function Get-Job([string]$jobFilePath) { + $jobString = Get-Content -Path $jobFilePath + $job = (ConvertFrom-Json -InputObject $jobString).job + return $job +} + +function Install-Sdks([string]$jobFilePath, [string]$repoContentsPath, [string]$dotnetInstallScriptPath, [string]$dotnetInstallDir) { + $job = Get-Job -jobFilePath $jobFilePath + + $installedSdks = dotnet --list-sdks | ForEach-Object { $_.Split(' ')[0] } + if ($installedSdks.GetType().Name -eq "String") { + # if only a single value was returned (expected in the container), then force it to an array + $installedSdks = @($installedSdks) + } + Write-Host "Currently installed SDKs: $installedSdks" + $rootDir = Convert-Path $repoContentsPath + + $candidateDirectories = @() + if ("directory" -in $job.source.PSobject.Properties.Name) { + $candidateDirectories += $job.source.directory + } + if ("directories" -in $job.source.PSobject.Properties.Name) { + $candidateDirectories += $job.source.directories + } + + $globalJsonRelativePaths = Get-DirectoriesForSdkInstall ` + -repoRoot $rootDir ` + -updateDirectories $candidateDirectories + + foreach ($globalJsonRelativePath in $globalJsonRelativePaths) { + $globalJsonPath = "$rootDir/$globalJsonRelativePath" + $globalJson = Get-Content $globalJsonPath | ConvertFrom-Json + $sdkVersion = $globalJson.sdk.version + if (-Not ($sdkVersion -in $installedSdks)) { + $installedSdks += $sdkVersion + Write-Host "Installing SDK $sdkVersion as specified in $globalJsonRelativePath" + & $dotnetInstallScriptPath --version $sdkVersion --install-dir $dotnetInstallDir + } + } + + # report the final set + dotnet --list-sdks +} diff --git a/nuget/updater/install-sdks.ps1 b/nuget/updater/install-sdks.ps1 new file mode 100755 index 0000000000..5d3fbe1650 --- /dev/null +++ b/nuget/updater/install-sdks.ps1 @@ -0,0 +1,18 @@ +Set-StrictMode -version 2.0 +$ErrorActionPreference = "Stop" + +. $PSScriptRoot\common.ps1 + +try { + Install-Sdks ` + -jobFilePath $env:DEPENDABOT_JOB_PATH ` + -repoContentsPath $env:DEPENDABOT_REPO_CONTENTS_PATH ` + -dotnetInstallScriptPath $env:DOTNET_INSTALL_SCRIPT_PATH ` + -dotnetInstallDir $env:DOTNET_INSTALL_DIR +} +catch { + Write-Host $_ + Write-Host $_.Exception + Write-Host $_.ScriptStackTrace + exit 1 +} diff --git a/nuget/updater/main.ps1 b/nuget/updater/main.ps1 index bbb3299405..49fd2bc94e 100644 --- a/nuget/updater/main.ps1 +++ b/nuget/updater/main.ps1 @@ -4,8 +4,6 @@ $ErrorActionPreference = "Stop" . $PSScriptRoot\common.ps1 $updaterTool = "$env:DEPENDABOT_NATIVE_HELPERS_PATH/nuget/NuGetUpdater/NuGetUpdater.Cli" -$jobString = Get-Content -Path $env:DEPENDABOT_JOB_PATH -$job = (ConvertFrom-Json -InputObject $jobString).job # Function return values in PowerShell are wacky and contain all of the output produced during the function call. # Because of this, we need a reliable way to communicate _only_ the result of executing a single command, not its @@ -15,6 +13,7 @@ $job = (ConvertFrom-Json -InputObject $jobString).job $operationExitCode = 0 function Get-Files { + $job = Get-Job -jobFilePath $env:DEPENDABOT_JOB_PATH Write-Host "Job: $($job | ConvertTo-Json)" & $updaterTool clone ` --job-path $env:DEPENDABOT_JOB_PATH ` @@ -24,45 +23,13 @@ function Get-Files { $script:operationExitCode = $LASTEXITCODE } -function Install-Sdks { - $installedSdks = dotnet --list-sdks | ForEach-Object { $_.Split(' ')[0] } - if ($installedSdks.GetType().Name -eq "String") { - # if only a single value was returned (expected in the container), then force it to an array - $installedSdks = @($installedSdks) - } - Write-Host "Currently installed SDKs: $installedSdks" - $rootDir = Convert-Path $env:DEPENDABOT_REPO_CONTENTS_PATH - - $candidateDirectories = @() - if ("directory" -in $job.source.PSobject.Properties.Name) { - $candidateDirectories += $job.source.directory - } - if ("directories" -in $job.source.PSobject.Properties.Name) { - $candidateDirectories += $job.source.directories - } - - $globalJsonRelativePaths = Get-DirectoriesForSdkInstall ` - -repoRoot $rootDir ` - -updateDirectories $candidateDirectories - - foreach ($globalJsonRelativePath in $globalJsonRelativePaths) { - $globalJsonPath = "$rootDir/$globalJsonRelativePath" - $globalJson = Get-Content $globalJsonPath | ConvertFrom-Json - $sdkVersion = $globalJson.sdk.version - if (-Not ($sdkVersion -in $installedSdks)) { - $installedSdks += $sdkVersion - Write-Host "Installing SDK $sdkVersion as specified in $globalJsonRelativePath" - & $env:DOTNET_INSTALL_SCRIPT_PATH --version $sdkVersion --install-dir $env:DOTNET_INSTALL_DIR - } - } - - # report the final set - dotnet --list-sdks -} - function Update-Files { # install relevant SDKs - Install-Sdks + Install-Sdks ` + -jobFilePath $env:DEPENDABOT_JOB_PATH ` + -repoContentsPath $env:DEPENDABOT_REPO_CONTENTS_PATH ` + -dotnetInstallScriptPath $env:DOTNET_INSTALL_SCRIPT_PATH ` + -dotnetInstallDir $env:DOTNET_INSTALL_DIR # TODO: install workloads? Push-Location $env:DEPENDABOT_REPO_CONTENTS_PATH