Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Parallelize tests #17872

Draft
wants to merge 16 commits into
base: main
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
104 changes: 52 additions & 52 deletions eng/Build.ps1

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion eng/Versions.props
Original file line number Diff line number Diff line change
Expand Up @@ -184,7 +184,7 @@
<MicrosoftNETCoreAppRefVersion>3.1.0</MicrosoftNETCoreAppRefVersion>
<MicrosoftNETCoreILDAsmVersion>5.0.0-preview.7.20364.11</MicrosoftNETCoreILDAsmVersion>
<MicrosoftNETCoreILAsmVersion>5.0.0-preview.7.20364.11</MicrosoftNETCoreILAsmVersion>
<MicrosoftNETTestSdkVersion>17.4.0</MicrosoftNETTestSdkVersion>
<MicrosoftNETTestSdkVersion>17.11.1</MicrosoftNETTestSdkVersion>
<NewtonsoftJsonVersion>13.0.3</NewtonsoftJsonVersion>
<RoslynToolsSignToolVersion>1.0.0-beta2-dev3</RoslynToolsSignToolVersion>
<StreamJsonRpcVersion>2.18.48</StreamJsonRpcVersion>
Expand Down
3 changes: 2 additions & 1 deletion eng/build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -214,7 +214,8 @@ function Test() {
projectname=$(basename -- "$testproject")
projectname="${projectname%.*}"
testlogpath="$artifacts_dir/TestResults/$configuration/${projectname}_$targetframework.xml"
args="test \"$testproject\" --no-restore --no-build -c $configuration -f $targetframework --test-adapter-path . --logger \"xunit;LogFilePath=$testlogpath\" --blame --results-directory $artifacts_dir/TestResults/$configuration -p:vstestusemsbuildoutput=false"
args="test \"$testproject\" --no-restore --no-build -c $configuration -f $targetframework --test-adapter-path . --logger \"xunit;LogFilePath=$testlogpath\" --blame-hang-timeout 5minutes --results-directory $artifacts_dir/TestResults/$configuration -p:vstestusemsbuildoutput=false"
args+=" -- xUnit.MaxParallelThreads=4"
"$DOTNET_INSTALL_DIR/dotnet" $args || exit $?
}

Expand Down
2 changes: 1 addition & 1 deletion src/Compiler/Utilities/illib.fs
Original file line number Diff line number Diff line change
Expand Up @@ -137,7 +137,7 @@ module internal PervasiveAutoOpens =
type Async with

static member RunImmediate(computation: Async<'T>, ?cancellationToken) =
let cancellationToken = defaultArg cancellationToken Async.DefaultCancellationToken
let cancellationToken = defaultArg cancellationToken CancellationToken.None
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We probably never want the default token to cancel compiler jobs.


let ts = TaskCompletionSource<'T>()

Expand Down
7 changes: 4 additions & 3 deletions src/FSharp.Core/mailbox.fs
Original file line number Diff line number Diff line change
Expand Up @@ -340,10 +340,11 @@ type Mailbox<'Msg>(cancellationSupported: bool, isThrowExceptionAfterDisposed: b
inboxStore.Clear()

arrivals.Clear()
isDisposed <- true)
isDisposed <- true

if isNotNull pulse then
(pulse :> IDisposable).Dispose()
if isNotNull pulse then
(pulse :> IDisposable).Dispose()
pulse <- null)
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is a provisional fix for #17849 This is a core change that should be extracted to a PR and carefully thought through.


#if DEBUG
member x.UnsafeContents = (x.inbox, arrivals, pulse, savedCont) |> box
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
<PackagePath>content\myfiles\</PackagePath>
</Content>
<PackageReference Include="BasicProvider" Version="1.0.0" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.3.1" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.11.1" />
</ItemGroup>

<Target Name="RemovePackagesFromCache" BeforeTargets="Restore">
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,8 @@
</None>

<PackageReference Include="ComboProvider" Version="1.0.0" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.3.1" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.11.1" />

</ItemGroup>

<Target Name="RemovePackagesFromCache" BeforeTargets="Restore">
Expand Down
11 changes: 10 additions & 1 deletion tests/FSharp.Build.UnitTests/FSharp.Build.UnitTests.fsproj
Original file line number Diff line number Diff line change
Expand Up @@ -4,17 +4,26 @@

<PropertyGroup>
<TargetFrameworks>net472;$(FSharpNetCoreProductTargetFramework)</TargetFrameworks>
<TargetFrameworks Condition="'$(OS)' == 'Unix'">$(FSharpNetCoreProductTargetFramework)</TargetFrameworks>
<TargetFrameworks Condition="'$(OS)' == 'Unix' or '$(BUILDING_USING_DOTNET)' == 'true'">$(FSharpNetCoreProductTargetFramework)</TargetFrameworks>
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

To make the whole FSharp.sln work with BUILDING_USING_DOTNET

<OutputType>Library</OutputType>
<DisableImplicitFSharpCoreReference>true</DisableImplicitFSharpCoreReference>
<UnitTestType>xunit</UnitTestType>
</PropertyGroup>

<ItemGroup>
<Compile Include="..\FSharp.Test.Utilities\XunitSetup.fs">
<Link>XunitSetup.fs</Link>
</Compile>
<Compile Include="WriteCodeFragmentTests.fs" />
<Compile Include="MapSourceRootsTests.fs" />
</ItemGroup>

<ItemGroup>
<Content Include="xunit.runner.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
</ItemGroup>

<ItemGroup>
<ProjectReference Include="$(FSharpSourcesRoot)\FSharp.Build\FSharp.Build.fsproj" />
<ProjectReference Include="$(FSharpSourcesRoot)\FSharp.Core\FSharp.Core.fsproj" />
Expand Down
5 changes: 5 additions & 0 deletions tests/FSharp.Build.UnitTests/xunit.runner.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
{
"$schema": "https://xunit.net/schema/current/xunit.runner.schema.json",
"appDomain": "denied",
"parallelizeAssembly": true
}
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ open FSharp.Test
open FSharp.Test.Compiler
open System

[<Collection(nameof DoNotRunInParallel)>]
module utf8output =

[<Fact>]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@ open FSharp.Test.Compiler
open System
open System.IO

// reportTime uses global state.
[<Collection(nameof DoNotRunInParallel)>]
module times =

// This test was automatically generated (moved from FSharpQA suite - CompilerOptions/fsc/times)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,5 +14,5 @@ let success =
asm.Version.Major = 1 &&
asm.Version.Minor = 2 &&
asm.Version.Build = 3 &&
(abs (asm.Version.Revision - (int defaultRevision))) < 10 // default value is seconds in the current day / 2. Check if within 10 sec of that.
(abs (asm.Version.Revision - (int defaultRevision))) < 60 // default value is seconds in the current day / 2. Check if within 60 sec of that.
if success then () else failwith "Failed: 1"
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ open Xunit
open FSharp.Test
open FSharp.Test.Compiler

[<Collection(nameof DoNotRunInParallel)>]
module Events =

let verifyCompile compilation =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -56,9 +56,9 @@ let ``Stackoverflow reproduction`` compilation =
| CompilationResult.Success ({OutputPath = Some dllFile} as s) ->
let fsharpCoreFile = typeof<voption<_>>.Assembly.Location
File.Copy(fsharpCoreFile, Path.Combine(Path.GetDirectoryName(dllFile), Path.GetFileName(fsharpCoreFile)), true)
let _exitCode, _stdout, stderr, _exn = CompilerAssert.ExecuteAndReturnResult (dllFile, isFsx=false, deps = s.Dependencies, newProcess=true)
let result = CompilerAssert.ExecuteAndReturnResult (dllFile, isFsx=false, deps = s.Dependencies, newProcess=true)

Assert.True(stderr.Contains "stack overflow" || stderr.Contains "StackOverflow")
Assert.True(result.StdErr.Contains "stack overflow" || result.StdErr.Contains "StackOverflow")

| _ -> failwith (sprintf "%A" compilationResult)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,9 @@
<Compile Include="..\service\FsUnit.fs">
<Link>FsUnit.fs</Link>
</Compile>
<Compile Include="..\FSharp.Test.Utilities\XunitSetup.fs">
<Link>XunitSetup.fs</Link>
</Compile>
<Compile Include="CompilerDirectives\Line.fs" />
<Compile Include="CompilerDirectives\Ifdef.fs" />
<Compile Include="CompilerDirectives\NonStringArgs.fs" />
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ open FSharp.Compiler.Diagnostics

open Xunit

open FSharp.Test
open FSharp.Test.ProjectGeneration
open FSharp.Test.ProjectGeneration.Helpers
open System.IO
Expand Down Expand Up @@ -1064,67 +1065,71 @@ type private LoadClosureTestShim(currentFileSystem: IFileSystem) =
?shouldShadowCopy = shouldShadowCopy
)

[<Theory>]
[<InlineData(false)>]
[<InlineData(true)>]
let ``The script load closure should always be evaluated`` useTransparentCompiler =
async {
// The LoadScriptClosure uses the file system shim so we need to reset that.
let currentFileSystem = FileSystemAutoOpens.FileSystem
let assumeDotNetFramework =
// The old BackgroundCompiler uses assumeDotNetFramework = true
// This is not always correctly loading when this test runs on non-Windows.
if System.Runtime.InteropServices.RuntimeInformation.FrameworkDescription.StartsWith(".NET Framework") then
None
else
Some false
// Because it is mutating FileSystem!
[<Collection(nameof DoNotRunInParallel)>]
module TestsMutatingFileSystem =

[<Theory>]
[<InlineData(false)>]
[<InlineData(true)>]
let ``The script load closure should always be evaluated`` useTransparentCompiler =
async {
// The LoadScriptClosure uses the file system shim so we need to reset that.
let currentFileSystem = FileSystemAutoOpens.FileSystem
let assumeDotNetFramework =
// The old BackgroundCompiler uses assumeDotNetFramework = true
// This is not always correctly loading when this test runs on non-Windows.
if System.Runtime.InteropServices.RuntimeInformation.FrameworkDescription.StartsWith(".NET Framework") then
None
else
Some false

try
let checker = FSharpChecker.Create(useTransparentCompiler = useTransparentCompiler)
let fileSystemShim = LoadClosureTestShim(currentFileSystem)
// Override the file system shim for loading b.fsx
FileSystem <- fileSystemShim

let! initialSnapshot, _ =
checker.GetProjectSnapshotFromScript(
"a.fsx",
SourceTextNew.ofString fileSystemShim.aFsx,
documentSource = DocumentSource.Custom fileSystemShim.DocumentSource,
?assumeDotNetFramework = assumeDotNetFramework
)

// File b.fsx should also be included in the snapshot.
Assert.Equal(2, initialSnapshot.SourceFiles.Length)

let! checkResults = checker.ParseAndCheckFileInProject("a.fsx", initialSnapshot)

match snd checkResults with
| FSharpCheckFileAnswer.Aborted -> failwith "Did not expected FSharpCheckFileAnswer.Aborted"
| FSharpCheckFileAnswer.Succeeded checkFileResults -> Assert.Equal(0, checkFileResults.Diagnostics.Length)
try
let checker = FSharpChecker.Create(useTransparentCompiler = useTransparentCompiler)
let fileSystemShim = LoadClosureTestShim(currentFileSystem)
// Override the file system shim for loading b.fsx
FileSystem <- fileSystemShim

let! initialSnapshot, _ =
checker.GetProjectSnapshotFromScript(
"a.fsx",
SourceTextNew.ofString fileSystemShim.aFsx,
documentSource = DocumentSource.Custom fileSystemShim.DocumentSource,
?assumeDotNetFramework = assumeDotNetFramework
)

// File b.fsx should also be included in the snapshot.
Assert.Equal(2, initialSnapshot.SourceFiles.Length)

let! checkResults = checker.ParseAndCheckFileInProject("a.fsx", initialSnapshot)

match snd checkResults with
| FSharpCheckFileAnswer.Aborted -> failwith "Did not expected FSharpCheckFileAnswer.Aborted"
| FSharpCheckFileAnswer.Succeeded checkFileResults -> Assert.Equal(0, checkFileResults.Diagnostics.Length)

// Update b.fsx, it should now load c.fsx
fileSystemShim.UpdateB()

// The constructed key for the load closure will the exactly the same as the first GetProjectSnapshotFromScript call.
// However, a none cached version will be computed first in GetProjectSnapshotFromScript and update the cache afterwards.
let! secondSnapshot, _ =
checker.GetProjectSnapshotFromScript(
"a.fsx",
SourceTextNew.ofString fileSystemShim.aFsx,
documentSource = DocumentSource.Custom fileSystemShim.DocumentSource,
?assumeDotNetFramework = assumeDotNetFramework
)

Assert.Equal(3, secondSnapshot.SourceFiles.Length)

let! checkResults = checker.ParseAndCheckFileInProject("a.fsx", secondSnapshot)

match snd checkResults with
| FSharpCheckFileAnswer.Aborted -> failwith "Did not expected FSharpCheckFileAnswer.Aborted"
| FSharpCheckFileAnswer.Succeeded checkFileResults -> Assert.Equal(0, checkFileResults.Diagnostics.Length)
finally
FileSystemAutoOpens.FileSystem <- currentFileSystem
}
// Update b.fsx, it should now load c.fsx
fileSystemShim.UpdateB()

// The constructed key for the load closure will the exactly the same as the first GetProjectSnapshotFromScript call.
// However, a none cached version will be computed first in GetProjectSnapshotFromScript and update the cache afterwards.
let! secondSnapshot, _ =
checker.GetProjectSnapshotFromScript(
"a.fsx",
SourceTextNew.ofString fileSystemShim.aFsx,
documentSource = DocumentSource.Custom fileSystemShim.DocumentSource,
?assumeDotNetFramework = assumeDotNetFramework
)

Assert.Equal(3, secondSnapshot.SourceFiles.Length)

let! checkResults = checker.ParseAndCheckFileInProject("a.fsx", secondSnapshot)

match snd checkResults with
| FSharpCheckFileAnswer.Aborted -> failwith "Did not expected FSharpCheckFileAnswer.Aborted"
| FSharpCheckFileAnswer.Succeeded checkFileResults -> Assert.Equal(0, checkFileResults.Diagnostics.Length)
finally
FileSystemAutoOpens.FileSystem <- currentFileSystem
}

[<Fact>]
let ``Parsing without cache and without project snapshot`` () =
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
// Copyright (c) Microsoft Corporation. All Rights Reserved. See License.txt in the project root for license information.

// Because of shared fsi session.
[<FSharp.Test.RunInSequence>]
module Language.BooleanReturningAndReturnTypeDirectedPartialActivePatternTests

open Xunit
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
// Copyright (c) Microsoft Corporation. All Rights Reserved. See License.txt in the project root for license information.

// Run sequentially because of shared fsiSession.
[<FSharp.Test.RunInSequence>]
module Language.SequenceExpression.SequenceExpressionTests

open FSharp.Test
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,17 +31,20 @@ module ScriptRunner =
let cu = cu |> withDefines defaultDefines
match cu with
| FS fsSource ->
use capture = new TestConsole.ExecutionCapture()
let engine = createEngine (fsSource.Options |> Array.ofList,version)
let res = evalScriptFromDiskInSharedSession engine cu
match res with
| CompilationResult.Failure _ -> res
| CompilationResult.Success s ->
if engine.GetOutput().Contains "TEST PASSED OK" then
| CompilationResult.Success _ ->
if capture.OutText |> TestFramework.outputPassed then
res
else
failwith $"Results looked correct, but 'TEST PASSED OK' was not printed. Result: %A{s}"
failwith $"Results looked correct, but 'TEST PASSED OK' was not printed."

| _ -> failwith $"Compilation unit other than fsharp is not supported, cannot process %A{cu}"
| _ ->
printfn $"Cannot process %A{cu}"
failwith $"Compilation unit other than fsharp is not supported."

/// This test file was created by porting over (slower) FsharpSuite.Tests
/// In order to minimize human error, the test definitions have been copy-pasted and this adapter provides implementations of the test functions
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -93,14 +93,17 @@ let ``comprehensions-FSC_OPTIMIZED`` () = singleTestBuildAndRun "core/comprehens
[<Fact>]
let ``comprehensions-FSI`` () = singleTestBuildAndRun "core/comprehensions" FSI

[<Fact>]
let ``comprehensionshw-FSC_DEBUG`` () = singleTestBuildAndRun "core/comprehensions-hw" FSC_DEBUG
// Cancels default token.
[<Collection(nameof DoNotRunInParallel)>]
module Comprehensionshw =
[<Fact>]
let ``comprehensionshw-FSC_DEBUG`` () = singleTestBuildAndRun "core/comprehensions-hw" FSC_DEBUG

[<Fact>]
let ``comprehensionshw-FSC_OPTIMIZED`` () = singleTestBuildAndRun "core/comprehensions-hw" FSC_OPTIMIZED
[<Fact>]
let ``comprehensionshw-FSC_OPTIMIZED`` () = singleTestBuildAndRun "core/comprehensions-hw" FSC_OPTIMIZED

[<Fact>]
let ``comprehensionshw-FSI`` () = singleTestBuildAndRun "core/comprehensions-hw" FSI
[<Fact>]
let ``comprehensionshw-FSI`` () = singleTestBuildAndRun "core/comprehensions-hw" FSI

[<Fact>]
let ``genericmeasures-FSC_DEBUG`` () = singleTestBuildAndRun "core/genericmeasures" FSC_DEBUG
Expand Down Expand Up @@ -375,18 +378,21 @@ let ``recordResolution-FSC_OPTIMIZED`` () = singleTestBuildAndRun "core/recordRe
[<Fact>]
let ``recordResolution-FSI`` () = singleTestBuildAndRun "core/recordResolution" FSI

// This test has hardcoded expectations about current synchronization context
// Will be moved out of FsharpSuite.Tests in a later phase for desktop framework
[<FactForNETCOREAPP>]
let ``control-FSC_OPTIMIZED`` () = singleTestBuildAndRun "core/control" FSC_OPTIMIZED
// Cancels default token.
[<Collection(nameof DoNotRunInParallel)>]
module CoreControl =
// This test has hardcoded expectations about current synchronization context
// Will be moved out of FsharpSuite.Tests in a later phase for desktop framework
[<FactForNETCOREAPP>]
let ``control-FSC_OPTIMIZED`` () = singleTestBuildAndRun "core/control" FSC_OPTIMIZED

[<FactForNETCOREAPP>]
let ``control-FSI`` () = singleTestBuildAndRun "core/control" FSI
[<FactForNETCOREAPP>]
let ``control-FSI`` () = singleTestBuildAndRun "core/control" FSI

[<FactForNETCOREAPP>]
let ``control --tailcalls`` () =
let cfg = "core/control"
singleTestBuildAndRunAux cfg ["--tailcalls"] FSC_OPTIMIZED
[<FactForNETCOREAPP>]
let ``control --tailcalls`` () =
let cfg = "core/control"
singleTestBuildAndRunAux cfg ["--tailcalls"] FSC_OPTIMIZED

[<Fact>]
let ``controlChamenos-FSC_OPTIMIZED`` () =
Expand All @@ -401,7 +407,7 @@ let ``controlChamenos-FSI`` () =
[<Fact>]
let ``controlMailbox-FSC_OPTIMIZED`` () = singleTestBuildAndRun "core/controlMailbox" FSC_OPTIMIZED

[<Fact(Skip="Flaky")>]
[<Fact>]
let ``controlMailbox-FSI`` () = singleTestBuildAndRun "core/controlMailbox" FSI

[<Fact>]
Expand Down
Loading
Loading