Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Parallelize tests #17872

Draft
wants to merge 16 commits into
base: main
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from 5 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
119 changes: 90 additions & 29 deletions eng/Build.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -362,36 +362,81 @@ function VerifyAssemblyVersionsAndSymbols() {
}
}

function TestUsingMSBuild([string] $testProject, [string] $targetFramework, [string]$testadapterpath, [boolean] $asBackgroundJob = $false) {
function TestUsingMSBuild([string] $testProject, [string] $targetFramework, [string]$testadapterpath, [string] $backgroundJob = "", [string] $settings = "") {
$jobId = if ($backgroundJob) { "_$backgroundJob"} else {""}
$dotnetPath = InitializeDotNetCli
$dotnetExe = Join-Path $dotnetPath "dotnet.exe"
$projectName = [System.IO.Path]::GetFileNameWithoutExtension($testProject)
$testLogPath = "$ArtifactsDir\TestResults\$configuration\${projectName}_$targetFramework.xml"
$testBinLogPath = "$LogDir\${projectName}_$targetFramework.binlog"
$args = "test $testProject -c $configuration -f $targetFramework -v n --test-adapter-path $testadapterpath --logger ""xunit;LogFilePath=$testLogPath"" /bl:$testBinLogPath"
$args += " --blame --results-directory $ArtifactsDir\TestResults\$configuration -p:vstestusemsbuildoutput=false"
$testLogPath = "$ArtifactsDir\TestResults\$configuration\${projectName}_$targetFramework$jobId.xml"
$testBinLogPath = "$LogDir\${projectName}_$targetFramework$jobId.binlog"
$arguments = "test $testProject -c $configuration -f $targetFramework -v n --test-adapter-path $testadapterpath --logger ""xunit;LogFilePath=$testLogPath"" /bl:$testBinLogPath"
$arguments += " --blame --blame-hang-timeout 5minutes --results-directory $ArtifactsDir\TestResults\$configuration -p:vstestusemsbuildoutput=true"

if (-not $noVisualStudio -or $norestore) {
$args += " --no-restore"
$arguments += " --no-restore"
}

if (-not $noVisualStudio) {
$args += " --no-build"
$arguments += " --no-build"
}

if ($asBackgroundJob) {
Write-Host("Starting on the background: $args")
$arguments += " $settings"

if ($backgroundJob) {
Write-Host
Write-Host("Starting on the background: $arguments")
Write-Host("------------------------------------")
$bgJob = Start-Job -ScriptBlock {
& $using:dotnetExe test $using:testProject -c $using:configuration -f $using:targetFramework -v n --test-adapter-path $using:testadapterpath --logger "xunit;LogFilePath=$using:testLogPath" /bl:$using:testBinLogPath --blame --results-directory $using:ArtifactsDir\TestResults\$using:configuration
Start-Job -ScriptBlock {
$argArray = $using:arguments -Split " "
& $using:dotnetExe $argArray
if ($LASTEXITCODE -ne 0) {
throw "Command failed to execute with exit code $($LASTEXITCODE): $using:dotnetExe $using:args"
}
}
return $bgJob
} else{
Write-Host("$args")
Exec-Console $dotnetExe $args
} else {
Write-Host("$arguments")
Exec-Console $dotnetExe $arguments
}
}

function TestSolutionUsingMSBuild([string] $testSolution, [string] $targetFramework, [string] $testadapterpath, [string] $backgroundJob = "", [string] $settings = "") {
$jobId = if ($backgroundJob) { "_$backgroundJob"} else {""}
$dotnetPath = InitializeDotNetCli
$dotnetExe = Join-Path $dotnetPath "dotnet.exe"
$solutionName = [System.IO.Path]::GetFileNameWithoutExtension($testSolution)
$testLogPath = "$ArtifactsDir\TestResults\$configuration\{assembly}.{framework}$jobId.xml"
$testBinLogPath = "$LogDir\${solutionName}_$targetFramework$jobId.binlog"

$arguments = "test"

$arguments += " $testSolution -c $configuration -f $targetFramework --test-adapter-path $testadapterpath -v n --logger ""xunit;LogFilePath=$testLogPath"" /bl:$testBinLogPath"
$arguments += " --blame-hang-timeout 5minutes --results-directory $ArtifactsDir\TestResults\$configuration /p:VsTestUseMSBuildOutput=true"

if (-not $noVisualStudio -or $norestore) {
$arguments += " --no-restore"
}

if (-not $noVisualStudio) {
$arguments += " --no-build"
}

$arguments += " $settings"

if ($backgroundJob) {
Write-Host
Write-Host("Starting on the background: $arguments")
Write-Host("------------------------------------")
Start-Job -ScriptBlock {
$argArray = $using:arguments -Split " "
$argArray += "--no-build"
& $using:dotnetExe $argArray
if ($LASTEXITCODE -ne 0) {
throw "Command failed to execute with exit code $($LASTEXITCODE): $using:dotnetExe $using:args"
}
}
} else {
Write-Host("$arguments")
Exec-Console $dotnetExe $arguments
}
}

Expand Down Expand Up @@ -588,32 +633,48 @@ try {
$script:BuildCategory = "Test"
$script:BuildMessage = "Failure running tests"

function Receive($job) {
while($job.HasMoreData) {
Receive-Job $job | Write-Host
Start-Sleep -Seconds 1
}
Receive-Job $job -Wait -ErrorAction Stop
}

if ($testCoreClr) {
$bgJob = TestUsingMSBuild -testProject "$RepoRoot\tests\fsharp\FSharpSuite.Tests.fsproj" -targetFramework $script:coreclrTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharpSuite.Tests\" -asBackgroundJob $true
$cpuLimit = if ($ci) { "-m:2 -- xUnit.MaxParallelThreads=0.25x" } else { "" }
TestSolutionUsingMSBuild -testSolution "$RepoRoot\FSharp.sln" -targetFramework $script:coreclrTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Compiler.ComponentTests\" -settings $cpuLimit
}

TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Compiler.ComponentTests\FSharp.Compiler.ComponentTests.fsproj" -targetFramework $script:coreclrTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Compiler.ComponentTests\"
TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Compiler.Service.Tests\FSharp.Compiler.Service.Tests.fsproj" -targetFramework $script:coreclrTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Compiler.Service.Tests\"
TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Compiler.Private.Scripting.UnitTests\FSharp.Compiler.Private.Scripting.UnitTests.fsproj" -targetFramework $script:coreclrTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Compiler.Private.Scripting.UnitTests\"
TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Build.UnitTests\FSharp.Build.UnitTests.fsproj" -targetFramework $script:coreclrTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Build.UnitTests\"
TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Core.UnitTests\FSharp.Core.UnitTests.fsproj" -targetFramework $script:coreclrTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Core.UnitTests\"
if ($testDesktop -and -not $ci) {

# Split ComponentTests into processes using filter, because it is slow and underutilizes CPU locally.
$bgJob1 = TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Compiler.ComponentTests\FSharp.Compiler.ComponentTests.fsproj" -targetFramework $script:desktopTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Compiler.ComponentTests\" -backgroundJob 1 -settings "--filter ExecutionNode=1"
$bgJob2 = TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Compiler.ComponentTests\FSharp.Compiler.ComponentTests.fsproj" -targetFramework $script:desktopTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Compiler.ComponentTests\" -backgroundJob 2 -settings "--filter ExecutionNode=2"
$bgJob3 = TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Compiler.ComponentTests\FSharp.Compiler.ComponentTests.fsproj" -targetFramework $script:desktopTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Compiler.ComponentTests\" -backgroundJob 3 -settings "--filter ExecutionNode=3"
$bgJob4 = TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Compiler.ComponentTests\FSharp.Compiler.ComponentTests.fsproj" -targetFramework $script:desktopTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Compiler.ComponentTests\" -backgroundJob 4 -settings "--filter ExecutionNode=4"

TestSolutionUsingMSBuild -testSolution "$RepoRoot\FSharp.sln" -targetFramework $script:desktopTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Compiler.ComponentTests\" -settings " --filter ""Project!=FSharpSuite.Tests&Project!=FSharp.Compiler.ComponentTests"" "
# FSharpSuite does most of it's work in external processes, saturating the CPU. It makes sense to run it separately.
TestUsingMSBuild -testProject "$RepoRoot\tests\fsharp\FSharpSuite.Tests.fsproj" -targetFramework $script:desktopTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharpSuite.Tests\"

# Collect output from background jobs
Wait-job $bgJob | out-null
Receive-Job $bgJob -ErrorAction Stop
# Collect output from background jobs
Receive -job $bgJob1
Receive -job $bgJob2
Receive -job $bgJob3
Receive -job $bgJob4
}

if ($testDesktop) {
$bgJob = TestUsingMSBuild -testProject "$RepoRoot\tests\fsharp\FSharpSuite.Tests.fsproj" -targetFramework $script:desktopTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharpSuite.Tests\" -asBackgroundJob $true
if ($testDesktop -and $ci) {
$bgJob = TestUsingMSBuild -testProject "$RepoRoot\tests\fsharp\FSharpSuite.Tests.fsproj" -targetFramework $script:desktopTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharpSuite.Tests\" -backgroundJob 1

TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Compiler.ComponentTests\FSharp.Compiler.ComponentTests.fsproj" -targetFramework $script:desktopTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Compiler.ComponentTests\"
TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Compiler.Service.Tests\FSharp.Compiler.Service.Tests.fsproj" -targetFramework $script:desktopTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Compiler.Service.Tests\"
TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Compiler.Private.Scripting.UnitTests\FSharp.Compiler.Private.Scripting.UnitTests.fsproj" -targetFramework $script:desktopTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Compiler.Private.Scripting.UnitTests\"
TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Build.UnitTests\FSharp.Build.UnitTests.fsproj" -targetFramework $script:desktopTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Build.UnitTests\"
TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Core.UnitTests\FSharp.Core.UnitTests.fsproj" -targetFramework $script:desktopTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Core.UnitTests\"

# Collect output from background jobs
Wait-job $bgJob | out-null
Receive-Job $bgJob -ErrorAction Stop
Receive -job $bgJob
}

if ($testFSharpQA) {
Expand Down
2 changes: 1 addition & 1 deletion eng/Versions.props
Original file line number Diff line number Diff line change
Expand Up @@ -184,7 +184,7 @@
<MicrosoftNETCoreAppRefVersion>3.1.0</MicrosoftNETCoreAppRefVersion>
<MicrosoftNETCoreILDAsmVersion>5.0.0-preview.7.20364.11</MicrosoftNETCoreILDAsmVersion>
<MicrosoftNETCoreILAsmVersion>5.0.0-preview.7.20364.11</MicrosoftNETCoreILAsmVersion>
<MicrosoftNETTestSdkVersion>17.4.0</MicrosoftNETTestSdkVersion>
<MicrosoftNETTestSdkVersion>17.11.1</MicrosoftNETTestSdkVersion>
<NewtonsoftJsonVersion>13.0.3</NewtonsoftJsonVersion>
<RoslynToolsSignToolVersion>1.0.0-beta2-dev3</RoslynToolsSignToolVersion>
<StreamJsonRpcVersion>2.18.48</StreamJsonRpcVersion>
Expand Down
3 changes: 2 additions & 1 deletion eng/build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -214,7 +214,8 @@ function Test() {
projectname=$(basename -- "$testproject")
projectname="${projectname%.*}"
testlogpath="$artifacts_dir/TestResults/$configuration/${projectname}_$targetframework.xml"
args="test \"$testproject\" --no-restore --no-build -c $configuration -f $targetframework --test-adapter-path . --logger \"xunit;LogFilePath=$testlogpath\" --blame --results-directory $artifacts_dir/TestResults/$configuration -p:vstestusemsbuildoutput=false"
args="test \"$testproject\" --no-restore --no-build -c $configuration -f $targetframework --test-adapter-path . --logger \"xunit;LogFilePath=$testlogpath\" --blame-hang-timeout 5minutes --results-directory $artifacts_dir/TestResults/$configuration -p:vstestusemsbuildoutput=false"
args+=" -- xUnit.MaxParallelThreads=4"
"$DOTNET_INSTALL_DIR/dotnet" $args || exit $?
}

Expand Down
1 change: 1 addition & 0 deletions src/Compiler/Driver/GraphChecking/GraphProcessing.fs
Original file line number Diff line number Diff line change
Expand Up @@ -252,6 +252,7 @@ let processGraphAsync<'Item, 'Result when 'Item: equality and 'Item: comparison>
let rec queueNode node =
Async.Start(
async {
use! _catch = Async.OnCancel(completionSignal.TrySetCanceled >> ignore)
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fix. This was starting with a cancellation token but without catching the OCE.

let! res = processNode node |> Async.Catch

match res with
Expand Down
2 changes: 1 addition & 1 deletion src/Compiler/Utilities/illib.fs
Original file line number Diff line number Diff line change
Expand Up @@ -137,7 +137,7 @@ module internal PervasiveAutoOpens =
type Async with

static member RunImmediate(computation: Async<'T>, ?cancellationToken) =
let cancellationToken = defaultArg cancellationToken Async.DefaultCancellationToken
let cancellationToken = defaultArg cancellationToken CancellationToken.None
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We probably never want the default token to cancel compiler jobs.


let ts = TaskCompletionSource<'T>()

Expand Down
7 changes: 4 additions & 3 deletions src/FSharp.Core/mailbox.fs
Original file line number Diff line number Diff line change
Expand Up @@ -340,10 +340,11 @@ type Mailbox<'Msg>(cancellationSupported: bool, isThrowExceptionAfterDisposed: b
inboxStore.Clear()

arrivals.Clear()
isDisposed <- true)
isDisposed <- true

if isNotNull pulse then
(pulse :> IDisposable).Dispose()
if isNotNull pulse then
(pulse :> IDisposable).Dispose()
pulse <- null)
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is a provisional fix for #17849 This is a core change that should be extracted to a PR and carefully thought through.


#if DEBUG
member x.UnsafeContents = (x.inbox, arrivals, pulse, savedCont) |> box
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
<PackagePath>content\myfiles\</PackagePath>
</Content>
<PackageReference Include="BasicProvider" Version="1.0.0" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.3.1" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.11.1" />
</ItemGroup>

<Target Name="RemovePackagesFromCache" BeforeTargets="Restore">
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,8 @@
</None>

<PackageReference Include="ComboProvider" Version="1.0.0" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.3.1" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.11.1" />

</ItemGroup>

<Target Name="RemovePackagesFromCache" BeforeTargets="Restore">
Expand Down
11 changes: 10 additions & 1 deletion tests/FSharp.Build.UnitTests/FSharp.Build.UnitTests.fsproj
Original file line number Diff line number Diff line change
Expand Up @@ -4,17 +4,26 @@

<PropertyGroup>
<TargetFrameworks>net472;$(FSharpNetCoreProductTargetFramework)</TargetFrameworks>
<TargetFrameworks Condition="'$(OS)' == 'Unix'">$(FSharpNetCoreProductTargetFramework)</TargetFrameworks>
<TargetFrameworks Condition="'$(OS)' == 'Unix' or '$(BUILDING_USING_DOTNET)' == 'true'">$(FSharpNetCoreProductTargetFramework)</TargetFrameworks>
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

To make the whole FSharp.sln work with BUILDING_USING_DOTNET

<OutputType>Library</OutputType>
<DisableImplicitFSharpCoreReference>true</DisableImplicitFSharpCoreReference>
<UnitTestType>xunit</UnitTestType>
</PropertyGroup>

<ItemGroup>
<Compile Include="..\FSharp.Test.Utilities\XunitSetup.fs">
<Link>XunitSetup.fs</Link>
</Compile>
<Compile Include="WriteCodeFragmentTests.fs" />
<Compile Include="MapSourceRootsTests.fs" />
</ItemGroup>

<ItemGroup>
<Content Include="xunit.runner.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
</ItemGroup>

<ItemGroup>
<ProjectReference Include="$(FSharpSourcesRoot)\FSharp.Build\FSharp.Build.fsproj" />
<ProjectReference Include="$(FSharpSourcesRoot)\FSharp.Core\FSharp.Core.fsproj" />
Expand Down
5 changes: 5 additions & 0 deletions tests/FSharp.Build.UnitTests/xunit.runner.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
{
"$schema": "https://xunit.net/schema/current/xunit.runner.schema.json",
"appDomain": "denied",
"parallelizeAssembly": true
}
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ open FSharp.Test
open FSharp.Test.Compiler
open System

[<Collection(nameof DoNotRunInParallel)>]
module utf8output =

[<Fact>]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@ open FSharp.Test.Compiler
open System
open System.IO

// reportTime uses global state.
[<Collection(nameof DoNotRunInParallel)>]
module times =

// This test was automatically generated (moved from FSharpQA suite - CompilerOptions/fsc/times)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,5 +14,5 @@ let success =
asm.Version.Major = 1 &&
asm.Version.Minor = 2 &&
asm.Version.Build = 3 &&
(abs (asm.Version.Revision - (int defaultRevision))) < 10 // default value is seconds in the current day / 2. Check if within 10 sec of that.
(abs (asm.Version.Revision - (int defaultRevision))) < 60 // default value is seconds in the current day / 2. Check if within 60 sec of that.
if success then () else failwith "Failed: 1"
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ open Xunit
open FSharp.Test
open FSharp.Test.Compiler

[<Collection(nameof DoNotRunInParallel)>]
module Events =

let verifyCompile compilation =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -56,9 +56,9 @@ let ``Stackoverflow reproduction`` compilation =
| CompilationResult.Success ({OutputPath = Some dllFile} as s) ->
let fsharpCoreFile = typeof<voption<_>>.Assembly.Location
File.Copy(fsharpCoreFile, Path.Combine(Path.GetDirectoryName(dllFile), Path.GetFileName(fsharpCoreFile)), true)
let _exitCode, _stdout, stderr, _exn = CompilerAssert.ExecuteAndReturnResult (dllFile, isFsx=false, deps = s.Dependencies, newProcess=true)
let result = CompilerAssert.ExecuteAndReturnResult (dllFile, isFsx=false, deps = s.Dependencies, newProcess=true)

Assert.True(stderr.Contains "stack overflow" || stderr.Contains "StackOverflow")
Assert.True(result.StdErr.Contains "stack overflow" || result.StdErr.Contains "StackOverflow")

| _ -> failwith (sprintf "%A" compilationResult)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,9 @@
<Compile Include="..\service\FsUnit.fs">
<Link>FsUnit.fs</Link>
</Compile>
<Compile Include="..\FSharp.Test.Utilities\XunitSetup.fs">
<Link>XunitSetup.fs</Link>
</Compile>
<Compile Include="CompilerDirectives\Line.fs" />
<Compile Include="CompilerDirectives\Ifdef.fs" />
<Compile Include="CompilerDirectives\NonStringArgs.fs" />
Expand Down
Loading
Loading