Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Reenable Transparent Compiler tests #17933

Draft
wants to merge 30 commits into
base: main
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from 24 commits
Commits
Show all changes
30 commits
Select commit Hold shift + click to select a range
0ed0442
rebase
majocha Oct 21, 2024
72ae0c6
Merge branch 'main' into parallel-tests
majocha Oct 23, 2024
612a7b3
Merge branch 'main' into parallel-tests
majocha Oct 23, 2024
f2795ee
merge main
majocha Oct 24, 2024
d384a06
Merge branch 'main' into parallel-tests
majocha Oct 24, 2024
7a94530
Merge remote-tracking branch 'dotnet/main' into parallel-tests
majocha Oct 25, 2024
e93a33e
orchestrate instead of Async.Sleep
majocha Oct 25, 2024
47d9cdc
xunit customizations behind conditional compilation
majocha Oct 25, 2024
4a5ba8f
Merge branch 'main' into parallel-tests
majocha Oct 25, 2024
7359bdf
Merge branch 'main' into parallel-tests
majocha Oct 27, 2024
83c5b98
make console capture explicit
majocha Oct 27, 2024
1ef5c8d
diff
majocha Oct 28, 2024
e665e79
try fix test
majocha Oct 28, 2024
efc4c28
lazy pattern
majocha Oct 28, 2024
1b00731
reenable extras
majocha Oct 28, 2024
8f82c8d
process graph with Async.Parallel
majocha Oct 28, 2024
c8567d3
reenable transparent compiler tests
majocha Oct 28, 2024
dc139f4
service tests had parallel disabled
majocha Oct 28, 2024
db139aa
merge
majocha Oct 29, 2024
17964e5
improve
majocha Oct 29, 2024
8d6290e
exclude perf test
majocha Oct 29, 2024
323ebed
format
majocha Oct 29, 2024
0ffca1f
wip
majocha Oct 29, 2024
40296d9
disable one problematic case pending investigation
majocha Oct 29, 2024
7b709b5
run ProjectAnalysisTests in sequence
majocha Oct 29, 2024
3168420
wip
majocha Oct 29, 2024
e159de9
run ModuleReaderCancellationTests in sequence
majocha Oct 29, 2024
fc164e9
make some MultiProjectAnalysisTests independent
majocha Oct 29, 2024
b2b69ea
Merge branch 'main' into reenable-tests
majocha Oct 29, 2024
dc0aeaa
try fix test
majocha Oct 29, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
104 changes: 52 additions & 52 deletions eng/Build.ps1

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion eng/Versions.props
Original file line number Diff line number Diff line change
Expand Up @@ -184,7 +184,7 @@
<MicrosoftNETCoreAppRefVersion>3.1.0</MicrosoftNETCoreAppRefVersion>
<MicrosoftNETCoreILDAsmVersion>5.0.0-preview.7.20364.11</MicrosoftNETCoreILDAsmVersion>
<MicrosoftNETCoreILAsmVersion>5.0.0-preview.7.20364.11</MicrosoftNETCoreILAsmVersion>
<MicrosoftNETTestSdkVersion>17.4.0</MicrosoftNETTestSdkVersion>
<MicrosoftNETTestSdkVersion>17.11.1</MicrosoftNETTestSdkVersion>
<NewtonsoftJsonVersion>13.0.3</NewtonsoftJsonVersion>
<RoslynToolsSignToolVersion>1.0.0-beta2-dev3</RoslynToolsSignToolVersion>
<StreamJsonRpcVersion>2.18.48</StreamJsonRpcVersion>
Expand Down
3 changes: 2 additions & 1 deletion eng/build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -214,7 +214,8 @@ function Test() {
projectname=$(basename -- "$testproject")
projectname="${projectname%.*}"
testlogpath="$artifacts_dir/TestResults/$configuration/${projectname}_$targetframework.xml"
args="test \"$testproject\" --no-restore --no-build -c $configuration -f $targetframework --test-adapter-path . --logger \"xunit;LogFilePath=$testlogpath\" --blame --results-directory $artifacts_dir/TestResults/$configuration -p:vstestusemsbuildoutput=false"
args="test \"$testproject\" --no-restore --no-build -c $configuration -f $targetframework --test-adapter-path . --logger \"xunit;LogFilePath=$testlogpath\" --blame-hang-timeout 5minutes --results-directory $artifacts_dir/TestResults/$configuration -p:vstestusemsbuildoutput=false"
args+=" -- xUnit.MaxParallelThreads=4"
"$DOTNET_INSTALL_DIR/dotnet" $args || exit $?
}

Expand Down
61 changes: 12 additions & 49 deletions src/Compiler/Driver/GraphChecking/GraphProcessing.fs
Original file line number Diff line number Diff line change
Expand Up @@ -176,15 +176,6 @@ let processGraphAsync<'Item, 'Result when 'Item: equality and 'Item: comparison>
async {
let transitiveDeps = graph |> Graph.transitive
let dependents = graph |> Graph.reverse
// Cancellation source used to signal either an exception in one of the items or end of processing.
let! parentCt = Async.CancellationToken
use localCts = new CancellationTokenSource()

let completionSignal = TaskCompletionSource()

use _ = parentCt.Register(fun () -> completionSignal.TrySetCanceled() |> ignore)

use cts = CancellationTokenSource.CreateLinkedTokenSource(parentCt, localCts.Token)

let makeNode (item: 'Item) : GraphNode<'Item, 'Result> =
let info =
Expand Down Expand Up @@ -228,43 +219,19 @@ let processGraphAsync<'Item, 'Result when 'Item: equality and 'Item: comparison>
|> Option.defaultWith (fun () -> failwith $"Results for item '{node.Info.Item}' are not yet available")
}

let processedCount = IncrementableInt(0)

let handleExn (item, ex: exn) =
try
localCts.Cancel()
with :? ObjectDisposedException ->
// If it's disposed already, it means that the processing has already finished, most likely due to cancellation or failure in another node.
()

match ex with
| :? OperationCanceledException -> completionSignal.TrySetCanceled()
| _ ->
completionSignal.TrySetException(
GraphProcessingException($"[*] Encountered exception when processing item '{item}': {ex.Message}", ex)
)
|> ignore

let incrementProcessedNodesCount () =
if processedCount.Increment() = nodes.Count then
completionSignal.TrySetResult() |> ignore

let rec queueNode node =
Async.Start(
async {
use! _catch = Async.OnCancel(completionSignal.TrySetCanceled >> ignore)
let! res = processNode node |> Async.Catch

match res with
| Choice1Of2() -> ()
| Choice2Of2 ex -> handleExn (node.Info.Item, ex)
},
cts.Token
)

and processNode (node: GraphNode<'Item, 'Result>) : Async<unit> =
async {
try
do! processNode node
with ex ->
return
raise (
GraphProcessingException($"[*] Encountered exception when processing item '{node.Info.Item}': {ex.Message}", ex)
)
}

and processNode (node: GraphNode<'Item, 'Result>) =
async {
let info = node.Info

let! singleRes = work getItemPublicNode info
Expand All @@ -280,14 +247,10 @@ let processGraphAsync<'Item, 'Result when 'Item: equality and 'Item: comparison>
// Note: We cannot read 'dependent.ProcessedDepsCount' again to avoid returning the same item multiple times.
pdc = dependent.Info.Deps.Length)

unblockedDependents |> Array.iter queueNode
incrementProcessedNodesCount ()
do! unblockedDependents |> Array.map queueNode |> Async.Parallel |> Async.Ignore
}

leaves |> Array.iter queueNode

// Wait for end of processing, an exception, or an external cancellation request.
do! completionSignal.Task |> Async.AwaitTask
do! leaves |> Array.map queueNode |> Async.Parallel |> Async.Ignore

// All calculations succeeded - extract the results and sort in input order.
return
Expand Down
2 changes: 1 addition & 1 deletion src/Compiler/Utilities/illib.fs
Original file line number Diff line number Diff line change
Expand Up @@ -137,7 +137,7 @@ module internal PervasiveAutoOpens =
type Async with

static member RunImmediate(computation: Async<'T>, ?cancellationToken) =
let cancellationToken = defaultArg cancellationToken Async.DefaultCancellationToken
let cancellationToken = defaultArg cancellationToken CancellationToken.None

let ts = TaskCompletionSource<'T>()

Expand Down
7 changes: 4 additions & 3 deletions src/FSharp.Core/mailbox.fs
Original file line number Diff line number Diff line change
Expand Up @@ -340,10 +340,11 @@ type Mailbox<'Msg>(cancellationSupported: bool, isThrowExceptionAfterDisposed: b
inboxStore.Clear()

arrivals.Clear()
isDisposed <- true)
isDisposed <- true

if isNotNull pulse then
(pulse :> IDisposable).Dispose()
if isNotNull pulse then
(pulse :> IDisposable).Dispose()
pulse <- null)

#if DEBUG
member x.UnsafeContents = (x.inbox, arrivals, pulse, savedCont) |> box
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
<PackagePath>content\myfiles\</PackagePath>
</Content>
<PackageReference Include="BasicProvider" Version="1.0.0" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.3.1" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.11.1" />
</ItemGroup>

<Target Name="RemovePackagesFromCache" BeforeTargets="Restore">
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,8 @@
</None>

<PackageReference Include="ComboProvider" Version="1.0.0" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.3.1" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.11.1" />

</ItemGroup>

<Target Name="RemovePackagesFromCache" BeforeTargets="Restore">
Expand Down
11 changes: 10 additions & 1 deletion tests/FSharp.Build.UnitTests/FSharp.Build.UnitTests.fsproj
Original file line number Diff line number Diff line change
Expand Up @@ -4,17 +4,26 @@

<PropertyGroup>
<TargetFrameworks>net472;$(FSharpNetCoreProductTargetFramework)</TargetFrameworks>
<TargetFrameworks Condition="'$(OS)' == 'Unix'">$(FSharpNetCoreProductTargetFramework)</TargetFrameworks>
<TargetFrameworks Condition="'$(OS)' == 'Unix' or '$(BUILDING_USING_DOTNET)' == 'true'">$(FSharpNetCoreProductTargetFramework)</TargetFrameworks>
<OutputType>Library</OutputType>
<DisableImplicitFSharpCoreReference>true</DisableImplicitFSharpCoreReference>
<UnitTestType>xunit</UnitTestType>
</PropertyGroup>

<ItemGroup>
<Compile Include="..\FSharp.Test.Utilities\XunitSetup.fs">
<Link>XunitSetup.fs</Link>
</Compile>
<Compile Include="WriteCodeFragmentTests.fs" />
<Compile Include="MapSourceRootsTests.fs" />
</ItemGroup>

<ItemGroup>
<Content Include="xunit.runner.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
</ItemGroup>

<ItemGroup>
<ProjectReference Include="$(FSharpSourcesRoot)\FSharp.Build\FSharp.Build.fsproj" />
<ProjectReference Include="$(FSharpSourcesRoot)\FSharp.Core\FSharp.Core.fsproj" />
Expand Down
5 changes: 5 additions & 0 deletions tests/FSharp.Build.UnitTests/xunit.runner.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
{
"$schema": "https://xunit.net/schema/current/xunit.runner.schema.json",
"appDomain": "denied",
"parallelizeAssembly": true
}
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ open FSharp.Test
open FSharp.Test.Compiler
open System

[<Collection(nameof DoNotRunInParallel)>]
module utf8output =

[<Fact>]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@ open FSharp.Test.Compiler
open System
open System.IO

// reportTime uses global state.
[<Collection(nameof DoNotRunInParallel)>]
module times =

// This test was automatically generated (moved from FSharpQA suite - CompilerOptions/fsc/times)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,5 +14,5 @@ let success =
asm.Version.Major = 1 &&
asm.Version.Minor = 2 &&
asm.Version.Build = 3 &&
(abs (asm.Version.Revision - (int defaultRevision))) < 10 // default value is seconds in the current day / 2. Check if within 10 sec of that.
(abs (asm.Version.Revision - (int defaultRevision))) < 60 // default value is seconds in the current day / 2. Check if within 60 sec of that.
if success then () else failwith "Failed: 1"
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ open Xunit
open FSharp.Test
open FSharp.Test.Compiler

[<Collection(nameof DoNotRunInParallel)>]
module Events =

let verifyCompile compilation =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -56,9 +56,9 @@ let ``Stackoverflow reproduction`` compilation =
| CompilationResult.Success ({OutputPath = Some dllFile} as s) ->
let fsharpCoreFile = typeof<voption<_>>.Assembly.Location
File.Copy(fsharpCoreFile, Path.Combine(Path.GetDirectoryName(dllFile), Path.GetFileName(fsharpCoreFile)), true)
let _exitCode, _stdout, stderr, _exn = CompilerAssert.ExecuteAndReturnResult (dllFile, isFsx=false, deps = s.Dependencies, newProcess=true)
let result = CompilerAssert.ExecuteAndReturnResult (dllFile, isFsx=false, deps = s.Dependencies, newProcess=true)

Assert.True(stderr.Contains "stack overflow" || stderr.Contains "StackOverflow")
Assert.True(result.StdErr.Contains "stack overflow" || result.StdErr.Contains "StackOverflow")

| _ -> failwith (sprintf "%A" compilationResult)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,9 @@
<Compile Include="..\service\FsUnit.fs">
<Link>FsUnit.fs</Link>
</Compile>
<Compile Include="..\FSharp.Test.Utilities\XunitSetup.fs">
<Link>XunitSetup.fs</Link>
</Compile>
<Compile Include="CompilerDirectives\Line.fs" />
<Compile Include="CompilerDirectives\Ifdef.fs" />
<Compile Include="CompilerDirectives\NonStringArgs.fs" />
Expand Down
Loading
Loading