From aa77540b7e0e05d7f31252c66d037cacd5c3c282 Mon Sep 17 00:00:00 2001 From: Martijn Laarman Date: Thu, 8 Feb 2024 16:25:49 +0100 Subject: [PATCH 1/5] extract more generic xunit project from Elastic.Elasticsearch.Xunit --- Elastic.Abstractions.sln | 7 + .../Elastic.Xunit.ExampleComplex/Setup.cs | 4 +- .../TestWithoutClusterFixture.cs | 1 - .../Elastic.Xunit.ExampleComplex/Tests.cs | 90 +++-- .../ExampleTest.cs | 2 +- examples/ScratchPad/Program.cs | 8 +- .../Elastic.Elasticsearch.Xunit.csproj | 1 + .../ElasticXunitConfigurationAttribute.cs | 39 +-- .../ElasticXunitRunOptions.cs | 48 +-- .../Sdk/ElasticTestFramework.cs | 22 +- .../Sdk/ElasticTestFrameworkDiscoverer.cs | 41 --- .../Sdk/ForEachAsyncExtensions.cs | 25 -- .../Sdk/TestAssemblyRunner.cs | 251 ++------------ .../Sdk/TestCollectionRunner.cs | 48 --- .../Sdk/TestFrameworkExecutor.cs | 91 ----- .../ElasticTestCaseDiscoverer.cs | 29 +- .../IntegrationTestClusterAttribute.cs | 23 -- .../IntegrationTestDiscoverer.cs | 23 +- .../XunitPlumbing/UnitTestDiscoverer.cs | 10 +- src/Elastic.Xunit/Elastic.Xunit.csproj | 14 + src/Elastic.Xunit/ForEachAsyncExtensions.cs | 24 ++ .../PartitioningConfigurationAttribute.cs | 46 +++ src/Elastic.Xunit/PartitioningRunOptions.cs | 63 ++++ .../PartitioningTestAssemblyRunner.cs | 327 ++++++++++++++++++ .../PartitioningTestFramework.cs | 27 ++ .../PartitioningTestFrameworkDiscoverer.cs | 33 ++ .../PartitioningTestFrameworkExecutor.cs | 80 +++++ src/Elastic.Xunit/TestCollectionRunner.cs | 47 +++ 28 files changed, 845 insertions(+), 579 deletions(-) delete mode 100644 src/Elastic.Elasticsearch.Xunit/Sdk/ElasticTestFrameworkDiscoverer.cs delete mode 100644 src/Elastic.Elasticsearch.Xunit/Sdk/ForEachAsyncExtensions.cs delete mode 100644 src/Elastic.Elasticsearch.Xunit/Sdk/TestCollectionRunner.cs delete mode 100644 src/Elastic.Elasticsearch.Xunit/Sdk/TestFrameworkExecutor.cs delete mode 100644 src/Elastic.Elasticsearch.Xunit/XunitPlumbing/IntegrationTestClusterAttribute.cs create mode 100644 src/Elastic.Xunit/Elastic.Xunit.csproj create mode 100644 src/Elastic.Xunit/ForEachAsyncExtensions.cs create mode 100644 src/Elastic.Xunit/PartitioningConfigurationAttribute.cs create mode 100644 src/Elastic.Xunit/PartitioningRunOptions.cs create mode 100644 src/Elastic.Xunit/PartitioningTestAssemblyRunner.cs create mode 100644 src/Elastic.Xunit/PartitioningTestFramework.cs create mode 100644 src/Elastic.Xunit/PartitioningTestFrameworkDiscoverer.cs create mode 100644 src/Elastic.Xunit/PartitioningTestFrameworkExecutor.cs create mode 100644 src/Elastic.Xunit/TestCollectionRunner.cs diff --git a/Elastic.Abstractions.sln b/Elastic.Abstractions.sln index 9cada97..412f115 100644 --- a/Elastic.Abstractions.sln +++ b/Elastic.Abstractions.sln @@ -45,6 +45,8 @@ EndProjectSection EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Elastic.Ephemeral.Example", "examples\Elastic.Ephemeral.Example\Elastic.Ephemeral.Example.csproj", "{9666AFDC-B0E8-489C-A25A-17E67303A969}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Elastic.Xunit", "src\Elastic.Xunit\Elastic.Xunit.csproj", "{4D3CA43F-1653-4A6B-A5AC-FC3DB32F1DDA}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -65,6 +67,7 @@ Global {C05F7B36-EEF7-4BCD-86A2-F5F1BB8CFEB9} = {77E78EDE-60D5-469A-B431-443A7966A243} {D6997ADC-E933-418E-831C-DE1A78897493} = {F75ACC18-D314-4F1F-88A3-2002EAC4E207} {9666AFDC-B0E8-489C-A25A-17E67303A969} = {9D154338-4AA8-40A9-A378-B27C05D45791} + {4D3CA43F-1653-4A6B-A5AC-FC3DB32F1DDA} = {77E78EDE-60D5-469A-B431-443A7966A243} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {AFADDCED-A7DD-43E7-B03C-27F57AC5C358} @@ -114,5 +117,9 @@ Global {9666AFDC-B0E8-489C-A25A-17E67303A969}.Debug|Any CPU.Build.0 = Debug|Any CPU {9666AFDC-B0E8-489C-A25A-17E67303A969}.Release|Any CPU.ActiveCfg = Release|Any CPU {9666AFDC-B0E8-489C-A25A-17E67303A969}.Release|Any CPU.Build.0 = Release|Any CPU + {4D3CA43F-1653-4A6B-A5AC-FC3DB32F1DDA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {4D3CA43F-1653-4A6B-A5AC-FC3DB32F1DDA}.Debug|Any CPU.Build.0 = Debug|Any CPU + {4D3CA43F-1653-4A6B-A5AC-FC3DB32F1DDA}.Release|Any CPU.ActiveCfg = Release|Any CPU + {4D3CA43F-1653-4A6B-A5AC-FC3DB32F1DDA}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection EndGlobal diff --git a/examples/Elastic.Xunit.ExampleComplex/Setup.cs b/examples/Elastic.Xunit.ExampleComplex/Setup.cs index 31266be..0fc7466 100644 --- a/examples/Elastic.Xunit.ExampleComplex/Setup.cs +++ b/examples/Elastic.Xunit.ExampleComplex/Setup.cs @@ -20,12 +20,12 @@ public class MyRunOptions : ElasticXunitRunOptions public MyRunOptions() { ClusterFilter = ""; - RunUnitTests = false; + RunUnitTests = true; RunIntegrationTests = true; IntegrationTestsMayUseAlreadyRunningNode = true; Version = TestVersion; } - public static ElasticVersion TestVersion { get; } = "8.0.0-SNAPSHOT"; + public static ElasticVersion TestVersion { get; } = "latest-8"; } } diff --git a/examples/Elastic.Xunit.ExampleComplex/TestWithoutClusterFixture.cs b/examples/Elastic.Xunit.ExampleComplex/TestWithoutClusterFixture.cs index 9594304..3fab22a 100644 --- a/examples/Elastic.Xunit.ExampleComplex/TestWithoutClusterFixture.cs +++ b/examples/Elastic.Xunit.ExampleComplex/TestWithoutClusterFixture.cs @@ -8,7 +8,6 @@ namespace Elastic.Xunit.ExampleComplex { - [IntegrationTestCluster(typeof(TestCluster))] [SkipVersion("<6.3.0", "")] public class TestWithoutClusterFixture { diff --git a/examples/Elastic.Xunit.ExampleComplex/Tests.cs b/examples/Elastic.Xunit.ExampleComplex/Tests.cs index 48a565e..8a97d8d 100644 --- a/examples/Elastic.Xunit.ExampleComplex/Tests.cs +++ b/examples/Elastic.Xunit.ExampleComplex/Tests.cs @@ -2,17 +2,17 @@ // Elasticsearch B.V licenses this file to you under the Apache 2.0 License. // See the LICENSE file in the project root for more information +using Elastic.Elasticsearch.Managed; using Elastic.Elasticsearch.Xunit.XunitPlumbing; using Elasticsearch.Net; using FluentAssertions; +using Xunit; namespace Elastic.Xunit.ExampleComplex { public class MyTestClass : ClusterTestClassBase { - public MyTestClass(TestCluster cluster) : base(cluster) - { - } + public MyTestClass(TestCluster cluster) : base(cluster) { } [I] public void SomeTest() @@ -22,34 +22,70 @@ public void SomeTest() info.IsValid.Should().BeTrue(); Client.CreateIndex("INASda"); + Client.LowLevel.Search(PostData.Serializable(new {query = new {query_string = 1}})); + } + } + public class Tests1 : ClusterTestClassBase + { + public Tests1(TestCluster cluster) : base(cluster) { } - Client.LowLevel.Search(PostData.Serializable(new {query = new {query_string = 1}})); + [U] public void Unit1Test() => (1 + 1).Should().Be(2); + [U] public void Unit1Test1() => (1 + 1).Should().Be(2); + [U] public void Unit1Test2() => (1 + 1).Should().Be(2); + [U] public void Unit1Test3() => (1 + 1).Should().Be(2); + [U] public void Unit1Test4() => (1 + 1).Should().Be(2); + [U] public void Unit1Test5() => (1 + 1).Should().Be(2); + [U] public void Unit1Test6() => (1 + 1).Should().Be(2); + } + + public class Tests3 + { + [U] public void Unit3Test() => (1 + 1).Should().Be(2); + [U] public void Unit3Test1() => (1 + 1).Should().Be(2); + [U] public void Unit3Test2() => (1 + 1).Should().Be(2); + [U] public void Unit3Test3() => (1 + 1).Should().Be(2); + [U] public void Unit3Test4() => (1 + 1).Should().Be(2); + [U] public void Unit3Test5() => (1 + 1).Should().Be(2); + [U] public void Unit3Test6() => (1 + 1).Should().Be(2); + } + + public class Tests2 : ClusterTestClassBase + { + public Tests2(TestCluster cluster) : base(cluster) { } + + [U] public void Unit2Test() => (1 + 1).Should().Be(2); + [U] public void Unit2Test1() => (1 + 1).Should().Be(2); + [U] public void Unit2Test2() => (1 + 1).Should().Be(2); + [U] public void Unit2Test3() => (1 + 1).Should().Be(2); + [U] public void Unit2Test4() => (1 + 1).Should().Be(2); + [U] public void Unit2Test5() => (1 + 1).Should().Be(2); + [U] public void Unit2Test6() => (1 + 1).Should().Be(2); + } + + public class MyGenericTestClass : ClusterTestClassBase + { + public MyGenericTestClass(TestGenericCluster cluster) : base(cluster) { } + + [I] public void SomeTest() + { + var info = Client.RootNodeInfo(); + + info.IsValid.Should().BeTrue(); } + [U] public void MyGenericUnitTest() => (1 + 1).Should().Be(2); + [U] public void MyGenericUnitTest1() => (1 + 1).Should().Be(2); + [U] public void MyGenericUnitTest2() => (1 + 1).Should().Be(2); + [U] public void MyGenericUnitTest3() => (1 + 1).Should().Be(2); + [U] public void MyGenericUnitTest4() => (1 + 1).Should().Be(2); + [U] public void MyGenericUnitTest5() => (1 + 1).Should().Be(2); + [U] public void MyGenericUnitTest6() => (1 + 1).Should().Be(2); } -// -// public class MyGenericTestClass : ClusterTestClassBase -// { -// public MyGenericTestClass(TestGenericCluster cluster) : base(cluster) { } -// -// [I] public void SomeTest() -// { -// var info = this.Client.RootNodeInfo(); -// -// info.IsValid.Should().BeTrue(); -// } -// [U] public void UnitTest() -// { -// (1 + 1).Should().Be(2); -// } -// } [SkipVersion("<6.2.0", "")] public class SkipTestClass : ClusterTestClassBase { - public SkipTestClass(TestGenericCluster cluster) : base(cluster) - { - } + public SkipTestClass(TestGenericCluster cluster) : base(cluster) { } [I] public void SomeTest() @@ -62,4 +98,12 @@ public void SomeTest() [U] public void UnitTest() => (1 + 1).Should().Be(2); } + + public class DirectInterfaceTests : IClusterFixture + { + public DirectInterfaceTests(TestGenericCluster cluster) { } + + [U] + public void DirectUnitTest() => (1 + 1).Should().Be(2); + } } diff --git a/examples/Elastic.Xunit.ExampleMinimal/ExampleTest.cs b/examples/Elastic.Xunit.ExampleMinimal/ExampleTest.cs index 0b44131..493522c 100644 --- a/examples/Elastic.Xunit.ExampleMinimal/ExampleTest.cs +++ b/examples/Elastic.Xunit.ExampleMinimal/ExampleTest.cs @@ -21,7 +21,7 @@ public class MyTestCluster : XunitClusterBase /// We pass our configuration instance to the base class. /// We only configure it to run version 6.2.3 here but lots of additional options are available. /// - public MyTestCluster() : base(new XunitClusterConfiguration("8.0.0-SNAPSHOT") { }) + public MyTestCluster() : base(new XunitClusterConfiguration("latest-8") { }) { } } diff --git a/examples/ScratchPad/Program.cs b/examples/ScratchPad/Program.cs index 5ff29df..dabd8eb 100644 --- a/examples/ScratchPad/Program.cs +++ b/examples/ScratchPad/Program.cs @@ -24,8 +24,8 @@ public static class Program public static int Main() { - //ResolveVersions(); - ManualConfigRun(); + ResolveVersions(); + //ManualConfigRun(); //ValidateCombinations.Run(); return 0; } @@ -87,8 +87,8 @@ private static void ResolveVersions() { var versions = new[] { - "8.0.0-SNAPSHOT", "7.0.0-beta1", "6.6.1", "latest-7", "latest", "7.0.0", "7.4.0-SNAPSHOT", - "957e3089:7.2.0", "latest-6" + "latest-8", "7.0.0-beta1", "6.6.1", "latest-7", "latest", "7.0.0", "7.4.0-SNAPSHOT", + "957e3089:7.2.0" }; //versions = new[] {"latest-7"}; var products = new Product[] diff --git a/src/Elastic.Elasticsearch.Xunit/Elastic.Elasticsearch.Xunit.csproj b/src/Elastic.Elasticsearch.Xunit/Elastic.Elasticsearch.Xunit.csproj index 9877920..236f8b0 100644 --- a/src/Elastic.Elasticsearch.Xunit/Elastic.Elasticsearch.Xunit.csproj +++ b/src/Elastic.Elasticsearch.Xunit/Elastic.Elasticsearch.Xunit.csproj @@ -10,5 +10,6 @@ + \ No newline at end of file diff --git a/src/Elastic.Elasticsearch.Xunit/ElasticXunitConfigurationAttribute.cs b/src/Elastic.Elasticsearch.Xunit/ElasticXunitConfigurationAttribute.cs index 328cee6..37b0ff4 100644 --- a/src/Elastic.Elasticsearch.Xunit/ElasticXunitConfigurationAttribute.cs +++ b/src/Elastic.Elasticsearch.Xunit/ElasticXunitConfigurationAttribute.cs @@ -3,31 +3,20 @@ // See the LICENSE file in the project root for more information using System; +using Elastic.Xunit; -namespace Elastic.Elasticsearch.Xunit -{ - /// - /// An assembly attribute that specifies the - /// for Xunit tests within the assembly. - /// - [AttributeUsage(AttributeTargets.Assembly)] - public class ElasticXunitConfigurationAttribute : Attribute - { - /// - /// Creates a new instance of - /// - /// - /// A type deriving from that specifies the run options - /// - public ElasticXunitConfigurationAttribute(Type type) - { - var options = Activator.CreateInstance(type) as ElasticXunitRunOptions; - Options = options ?? new ElasticXunitRunOptions(); - } +namespace Elastic.Elasticsearch.Xunit; - /// - /// The run options - /// - public ElasticXunitRunOptions Options { get; } - } +/// +/// An assembly attribute that specifies the +/// for Xunit tests within the assembly. +/// +[AttributeUsage(AttributeTargets.Assembly)] +public class ElasticXunitConfigurationAttribute : PartitioningConfigurationAttribute +{ + /// Creates a new instance of . + /// + /// A type deriving from that specifies the run options + /// + public ElasticXunitConfigurationAttribute(Type type) : base(type) { } } diff --git a/src/Elastic.Elasticsearch.Xunit/ElasticXunitRunOptions.cs b/src/Elastic.Elasticsearch.Xunit/ElasticXunitRunOptions.cs index 26fb4db..b756bba 100644 --- a/src/Elastic.Elasticsearch.Xunit/ElasticXunitRunOptions.cs +++ b/src/Elastic.Elasticsearch.Xunit/ElasticXunitRunOptions.cs @@ -8,13 +8,15 @@ using System.Diagnostics; using Elastic.Elasticsearch.Xunit.XunitPlumbing; using Elastic.Stack.ArtifactsApi; +using Elastic.Xunit; +using Xunit.Abstractions; namespace Elastic.Elasticsearch.Xunit { /// /// The Xunit test runner options /// - public class ElasticXunitRunOptions + public class ElasticXunitRunOptions : PartitioningRunOptions { /// /// Informs the runner whether we expect to run integration tests. Defaults to true @@ -34,17 +36,11 @@ public class ElasticXunitRunOptions /// public bool RunUnitTests { get; set; } - /// - /// A global test filter that can be used to only run certain tests. - /// Accepts a comma separated list of filters - /// - public string TestFilter { get; set; } - /// /// A global cluster filter that can be used to only run certain cluster's tests. /// Accepts a comma separated list of filters /// - public string ClusterFilter { get; set; } + public string ClusterFilter { get => GroupFilter; set => GroupFilter = value; } /// /// Informs the runner what version of Elasticsearch is under test. Required for @@ -52,22 +48,34 @@ public class ElasticXunitRunOptions /// public ElasticVersion Version { get; set; } - /// - /// Called when the tests have finished running successfully - /// - /// Per cluster timings of the total test time, including starting Elasticsearch - /// All collection of failed cluster, failed tests tuples - public virtual void OnTestsFinished(Dictionary runnerClusterTotals, - ConcurrentBag> runnerFailedCollections) + public override void SetOptions(ITestFrameworkDiscoveryOptions discoveryOptions) { + base.SetOptions(discoveryOptions); + discoveryOptions.SetValue(nameof(Version), Version); + discoveryOptions.SetValue(nameof(RunIntegrationTests), RunIntegrationTests); + discoveryOptions.SetValue( + nameof(IntegrationTestsMayUseAlreadyRunningNode), + IntegrationTestsMayUseAlreadyRunningNode + ); + discoveryOptions.SetValue(nameof(RunUnitTests), RunUnitTests); + discoveryOptions.SetValue(nameof(TestFilter), TestFilter); + discoveryOptions.SetValue(nameof(ClusterFilter), ClusterFilter); } - /// - /// Called before tests run. An ideal place to perform actions such as writing information to - /// . - /// - public virtual void OnBeforeTestsRun() + public override void SetOptions(ITestFrameworkExecutionOptions executionOptions) { + + base.SetOptions(executionOptions); + executionOptions.SetValue(nameof(Version), Version); + executionOptions.SetValue(nameof(RunIntegrationTests), RunIntegrationTests); + executionOptions.SetValue( + nameof(IntegrationTestsMayUseAlreadyRunningNode), + IntegrationTestsMayUseAlreadyRunningNode + ); + executionOptions.SetValue(nameof(RunUnitTests), RunUnitTests); + executionOptions.SetValue(nameof(TestFilter), TestFilter); + executionOptions.SetValue(nameof(ClusterFilter), ClusterFilter); + } } } diff --git a/src/Elastic.Elasticsearch.Xunit/Sdk/ElasticTestFramework.cs b/src/Elastic.Elasticsearch.Xunit/Sdk/ElasticTestFramework.cs index c329c03..c93c9e7 100644 --- a/src/Elastic.Elasticsearch.Xunit/Sdk/ElasticTestFramework.cs +++ b/src/Elastic.Elasticsearch.Xunit/Sdk/ElasticTestFramework.cs @@ -5,28 +5,12 @@ using System.Reflection; using Xunit.Abstractions; using Xunit.Sdk; +using Elastic.Xunit; namespace Elastic.Elasticsearch.Xunit.Sdk { - public class ElasticTestFramework : XunitTestFramework + public class ElasticTestFramework : PartitioningTestFramework { - public ElasticTestFramework(IMessageSink messageSink) : base(messageSink) - { - } - - protected override ITestFrameworkDiscoverer CreateDiscoverer(IAssemblyInfo assemblyInfo) => - new ElasticTestFrameworkDiscoverer(assemblyInfo, SourceInformationProvider, DiagnosticMessageSink); - - protected override ITestFrameworkExecutor CreateExecutor(AssemblyName assemblyName) - { - var assembly = Assembly.Load(assemblyName); - var options = assembly.GetCustomAttribute()?.Options ?? - new ElasticXunitRunOptions(); - - return new TestFrameworkExecutor(assemblyName, SourceInformationProvider, DiagnosticMessageSink) - { - Options = options - }; - } + public ElasticTestFramework(IMessageSink messageSink) : base(messageSink) { } } } diff --git a/src/Elastic.Elasticsearch.Xunit/Sdk/ElasticTestFrameworkDiscoverer.cs b/src/Elastic.Elasticsearch.Xunit/Sdk/ElasticTestFrameworkDiscoverer.cs deleted file mode 100644 index 9edc1e9..0000000 --- a/src/Elastic.Elasticsearch.Xunit/Sdk/ElasticTestFrameworkDiscoverer.cs +++ /dev/null @@ -1,41 +0,0 @@ -// Licensed to Elasticsearch B.V under one or more agreements. -// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. -// See the LICENSE file in the project root for more information - -using System.Reflection; -using Xunit.Abstractions; -using Xunit.Sdk; - -namespace Elastic.Elasticsearch.Xunit.Sdk -{ - public class ElasticTestFrameworkDiscoverer : XunitTestFrameworkDiscoverer - { - public ElasticTestFrameworkDiscoverer(IAssemblyInfo assemblyInfo, ISourceInformationProvider sourceProvider, - IMessageSink diagnosticMessageSink, IXunitTestCollectionFactory collectionFactory = null) : base( - assemblyInfo, sourceProvider, diagnosticMessageSink, collectionFactory) - { - var a = Assembly.Load(new AssemblyName(assemblyInfo.Name)); - var options = a.GetCustomAttribute()?.Options ?? - new ElasticXunitRunOptions(); - Options = options; - } - - /// - /// The options for - /// - public ElasticXunitRunOptions Options { get; } - - protected override bool FindTestsForType(ITestClass testClass, bool includeSourceInformation, - IMessageBus messageBus, ITestFrameworkDiscoveryOptions discoveryOptions) - { - discoveryOptions.SetValue(nameof(ElasticXunitRunOptions.Version), Options.Version); - discoveryOptions.SetValue(nameof(ElasticXunitRunOptions.RunIntegrationTests), Options.RunIntegrationTests); - discoveryOptions.SetValue(nameof(ElasticXunitRunOptions.IntegrationTestsMayUseAlreadyRunningNode), - Options.IntegrationTestsMayUseAlreadyRunningNode); - discoveryOptions.SetValue(nameof(ElasticXunitRunOptions.RunUnitTests), Options.RunUnitTests); - discoveryOptions.SetValue(nameof(ElasticXunitRunOptions.TestFilter), Options.TestFilter); - discoveryOptions.SetValue(nameof(ElasticXunitRunOptions.ClusterFilter), Options.ClusterFilter); - return base.FindTestsForType(testClass, includeSourceInformation, messageBus, discoveryOptions); - } - } -} diff --git a/src/Elastic.Elasticsearch.Xunit/Sdk/ForEachAsyncExtensions.cs b/src/Elastic.Elasticsearch.Xunit/Sdk/ForEachAsyncExtensions.cs deleted file mode 100644 index 87bf715..0000000 --- a/src/Elastic.Elasticsearch.Xunit/Sdk/ForEachAsyncExtensions.cs +++ /dev/null @@ -1,25 +0,0 @@ -// Licensed to Elasticsearch B.V under one or more agreements. -// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. -// See the LICENSE file in the project root for more information - -using System; -using System.Collections.Concurrent; -using System.Collections.Generic; -using System.Linq; -using System.Threading.Tasks; - -namespace Elastic.Elasticsearch.Xunit.Sdk -{ - internal static class ForEachAsyncExtensions - { - internal static Task ForEachAsync(this IEnumerable source, int dop, Func body) => - Task.WhenAll( - from partition in Partitioner.Create(source).GetPartitions(dop) - select Task.Run(async delegate - { - using (partition) - while (partition.MoveNext()) - await body(partition.Current).ConfigureAwait(false); - })); - } -} diff --git a/src/Elastic.Elasticsearch.Xunit/Sdk/TestAssemblyRunner.cs b/src/Elastic.Elasticsearch.Xunit/Sdk/TestAssemblyRunner.cs index 8bd70da..f910cc9 100644 --- a/src/Elastic.Elasticsearch.Xunit/Sdk/TestAssemblyRunner.cs +++ b/src/Elastic.Elasticsearch.Xunit/Sdk/TestAssemblyRunner.cs @@ -13,271 +13,84 @@ using Elastic.Elasticsearch.Ephemeral; using Elastic.Elasticsearch.Ephemeral.Tasks.ValidationTasks; using Elastic.Elasticsearch.Xunit.XunitPlumbing; +using Elastic.Xunit; using Xunit.Abstractions; using Xunit.Sdk; namespace Elastic.Elasticsearch.Xunit.Sdk { - internal class TestAssemblyRunner : XunitTestAssemblyRunner - { - private readonly Dictionary> _assemblyFixtureMappings = - new Dictionary>(); - - private readonly List, GroupedByCluster>> _grouped; + public class TestAssemblyRunnerFactory : ITestAssemblyRunnerFactory + { + public XunitTestAssemblyRunner Create(ITestAssembly testAssembly, IEnumerable testCases, + IMessageSink diagnosticMessageSink, + IMessageSink executionMessageSink, ITestFrameworkExecutionOptions executionOptions) => + new TestAssemblyRunner(testAssembly, testCases, diagnosticMessageSink, executionMessageSink, + executionOptions); + } + internal class TestAssemblyRunner + : PartitioningTestAssemblyRunner> + { public TestAssemblyRunner(ITestAssembly testAssembly, IEnumerable testCases, IMessageSink diagnosticMessageSink, IMessageSink executionMessageSink, ITestFrameworkExecutionOptions executionOptions) - : base(testAssembly, testCases, diagnosticMessageSink, executionMessageSink, executionOptions) + : base(testAssembly, testCases, diagnosticMessageSink, executionMessageSink, executionOptions, typeof(IClusterFixture<>)) { - var tests = OrderTestCollections(); RunIntegrationTests = executionOptions.GetValue(nameof(ElasticXunitRunOptions.RunIntegrationTests)); + RunUnitTests = executionOptions.GetValue(nameof(ElasticXunitRunOptions.RunUnitTests)); IntegrationTestsMayUseAlreadyRunningNode = executionOptions.GetValue(nameof(ElasticXunitRunOptions .IntegrationTestsMayUseAlreadyRunningNode)); - RunUnitTests = executionOptions.GetValue(nameof(ElasticXunitRunOptions.RunUnitTests)); - TestFilter = executionOptions.GetValue(nameof(ElasticXunitRunOptions.TestFilter)); - ClusterFilter = executionOptions.GetValue(nameof(ElasticXunitRunOptions.ClusterFilter)); - - //bit side effecty, sets up _assemblyFixtureMappings before possibly letting xunit do its regular concurrency thing - _grouped = (from c in tests - let cluster = ClusterFixture(c.Item2.First().TestMethod.TestClass) - let testcase = new GroupedByCluster {Collection = c.Item1, TestCases = c.Item2, Cluster = cluster} - group testcase by testcase.Cluster - into g - orderby g.Count() descending - select g).ToList(); } - public ConcurrentBag Summaries { get; } = new ConcurrentBag(); - - public ConcurrentBag> FailedCollections { get; } = - new ConcurrentBag>(); - - public Dictionary ClusterTotals { get; } = new Dictionary(); - private bool RunIntegrationTests { get; } private bool IntegrationTestsMayUseAlreadyRunningNode { get; } private bool RunUnitTests { get; } - private string TestFilter { get; } - private string ClusterFilter { get; } - - protected override Task RunTestCollectionAsync(IMessageBus b, ITestCollection c, - IEnumerable t, CancellationTokenSource s) - { - var aggregator = new ExceptionAggregator(Aggregator); - var fixtureObjects = new Dictionary(); - foreach (var kv in _assemblyFixtureMappings) fixtureObjects.Add(kv.Key, kv.Value); - return new TestCollectionRunner(fixtureObjects, c, t, DiagnosticMessageSink, b, TestCaseOrderer, aggregator, - s) - .RunAsync(); - } protected override async Task RunTestCollectionsAsync(IMessageBus messageBus, CancellationTokenSource cancellationTokenSource) { - //threading guess - var defaultMaxConcurrency = Environment.ProcessorCount * 4; - if (RunUnitTests && !RunIntegrationTests) - return await UnitTestPipeline(defaultMaxConcurrency, messageBus, cancellationTokenSource) + return await RunAllWithoutPartitionFixture(messageBus, cancellationTokenSource) .ConfigureAwait(false); - return await IntegrationPipeline(defaultMaxConcurrency, messageBus, cancellationTokenSource) + return await RunAllTests(messageBus, cancellationTokenSource) .ConfigureAwait(false); } - - private async Task UnitTestPipeline(int defaultMaxConcurrency, IMessageBus messageBus, - CancellationTokenSource ctx) + protected override async Task UseStateAndRun(IEphemeralCluster cluster, Func runGroup) { - //make sure all clusters go in started state (won't actually start clusters in unit test mode) - //foreach (var g in this._grouped) g.Key?.Start(); - - var testFilters = CreateTestFilters(TestFilter); - await _grouped.SelectMany(g => g) - .ForEachAsync(defaultMaxConcurrency, - async g => { await RunTestCollections(messageBus, ctx, g, testFilters).ConfigureAwait(false); }) - .ConfigureAwait(false); - //foreach (var g in this._grouped) g.Key?.Dispose(); - - return new RunSummary + using (cluster) { - Total = Summaries.Sum(s => s.Total), - Failed = Summaries.Sum(s => s.Failed), - Skipped = Summaries.Sum(s => s.Skipped) - }; - } - - private async Task IntegrationPipeline(int defaultMaxConcurrency, IMessageBus messageBus, - CancellationTokenSource ctx) - { - var testFilters = CreateTestFilters(TestFilter); - foreach (var group in _grouped) - { - ElasticXunitRunner.CurrentCluster = @group.Key; - if (@group.Key == null) - { - var testCount = @group.SelectMany(q => q.TestCases).Count(); - Console.WriteLine($" -> Several tests skipped because they have no cluster associated"); - Summaries.Add(new RunSummary {Total = testCount, Skipped = testCount}); - continue; - } - - var type = @group.Key.GetType(); - var clusterName = type.Name.Replace("Cluster", string.Empty) ?? "UNKNOWN"; - if (!MatchesClusterFilter(clusterName)) continue; - - var dop = @group.Key.ClusterConfiguration?.MaxConcurrency ?? defaultMaxConcurrency; - dop = dop <= 0 ? defaultMaxConcurrency : dop; - - var timeout = @group.Key.ClusterConfiguration?.Timeout ?? TimeSpan.FromMinutes(2); - - var skipReasons = @group.SelectMany(g => g.TestCases.Select(t => t.SkipReason)).ToList(); - var allSkipped = skipReasons.All(r => !string.IsNullOrWhiteSpace(r)); - if (allSkipped) - { - Console.WriteLine($" -> All tests from {clusterName} are skipped under the current configuration"); - Summaries.Add(new RunSummary {Total = skipReasons.Count, Skipped = skipReasons.Count}); - continue; - } - - ClusterTotals.Add(clusterName, Stopwatch.StartNew()); - - bool ValidateRunningVersion() - { - try - { - var t = new ValidateRunningVersion(); - t.Run(@group.Key); - return true; - } - catch (Exception) - { - return false; - } - } + ElasticXunitRunner.CurrentCluster = cluster; + var clusterConfiguration = cluster.ClusterConfiguration; + var timeout = clusterConfiguration?.Timeout ?? TimeSpan.FromMinutes(2); + if (!IntegrationTestsMayUseAlreadyRunningNode || !ValidateRunningVersion(cluster)) + cluster.Start(timeout); - using (@group.Key) - { - if (!IntegrationTestsMayUseAlreadyRunningNode || !ValidateRunningVersion()) - @group.Key?.Start(timeout); - - await @group.ForEachAsync(dop, - async g => - { - await RunTestCollections(messageBus, ctx, g, testFilters).ConfigureAwait(false); - }) - .ConfigureAwait(false); - } - - ClusterTotals[clusterName].Stop(); + await runGroup(clusterConfiguration?.MaxConcurrency).ConfigureAwait(false); } - - return new RunSummary - { - Total = Summaries.Sum(s => s.Total), - Failed = Summaries.Sum(s => s.Failed), - Skipped = Summaries.Sum(s => s.Skipped) - }; } - private async Task RunTestCollections(IMessageBus messageBus, CancellationTokenSource ctx, GroupedByCluster g, - string[] testFilters) + private static bool ValidateRunningVersion(IEphemeralCluster cluster) { - var test = g.Collection.DisplayName.Replace("Test collection for", string.Empty).Trim(); - if (!MatchesATestFilter(test, testFilters)) return; - if (testFilters.Length > 0) Console.WriteLine(" -> " + test); - try { - var summary = await RunTestCollectionAsync(messageBus, g.Collection, g.TestCases, ctx) - .ConfigureAwait(false); - var type = g.Cluster?.GetType(); - var clusterName = type?.Name.Replace("Cluster", "") ?? "UNKNOWN"; - if (summary.Failed > 0) - FailedCollections.Add(Tuple.Create(clusterName, test)); - Summaries.Add(summary); + var t = new ValidateRunningVersion(); + t.Run(cluster); + return true; } - catch (TaskCanceledException) + catch (Exception) { - // TODO: What should happen here? + return false; } } - private static string[] CreateTestFilters(string testFilters) => - testFilters?.Split(',').Select(s => s.Trim()).Where(s => !string.IsNullOrWhiteSpace(s)).ToArray() - ?? new string[] { }; + public static Type GetClusterFixtureType(ITypeInfo testClass) => + GetPartitionFixtureType(testClass, typeof(IClusterFixture<>)); - private static bool MatchesATestFilter(string test, IReadOnlyCollection testFilters) - { - if (testFilters.Count == 0 || string.IsNullOrWhiteSpace(test)) return true; - return testFilters - .Any(filter => test.IndexOf(filter, StringComparison.OrdinalIgnoreCase) >= 0); - } - private bool MatchesClusterFilter(string cluster) - { - if (string.IsNullOrWhiteSpace(cluster) || string.IsNullOrWhiteSpace(ClusterFilter)) return true; - return ClusterFilter - .Split(new[] {','}, StringSplitOptions.RemoveEmptyEntries) - .Select(c => c.Trim()) - .Any(c => cluster.IndexOf(c, StringComparison.OrdinalIgnoreCase) >= 0); - } - - private IEphemeralCluster ClusterFixture(ITestClass testMethodTestClass) - { - var clusterType = GetClusterForClass(testMethodTestClass.Class); - if (clusterType == null) return null; - - if (_assemblyFixtureMappings.TryGetValue(clusterType, out var cluster)) return cluster; - Aggregator.Run(() => - { - var o = Activator.CreateInstance(clusterType); - cluster = o as IEphemeralCluster; - }); - _assemblyFixtureMappings.Add(clusterType, cluster); - return cluster; - } - - public static bool IsAnIntegrationTestClusterType(Type type) => - typeof(XunitClusterBase).GetTypeInfo().IsAssignableFrom(type.GetTypeInfo()) - || IsSubclassOfRawGeneric(typeof(XunitClusterBase<>), type); - - public static Type GetClusterForClass(ITypeInfo testClass) => - GetClusterFromClassClusterFixture(testClass) ?? GetClusterFromIntegrationAttribute(testClass); - - private static Type GetClusterFromClassClusterFixture(ITypeInfo testClass) => ( - from i in testClass.Interfaces - where i.IsGenericType - from a in i.GetGenericArguments() - select a.ToRuntimeType() - ).FirstOrDefault(IsAnIntegrationTestClusterType); - - private static Type GetClusterFromIntegrationAttribute(ITypeInfo testClass) => - testClass.GetCustomAttributes(typeof(IntegrationTestClusterAttribute)) - .FirstOrDefault()?.GetNamedArgument(nameof(IntegrationTestClusterAttribute.ClusterType)); - - private static bool IsSubclassOfRawGeneric(Type generic, Type toCheck) - { - while (toCheck != null && toCheck != typeof(object)) - { - var cur = toCheck.GetTypeInfo().IsGenericType ? toCheck.GetGenericTypeDefinition() : toCheck; - if (generic == cur) return true; - - toCheck = toCheck.GetTypeInfo().BaseType; - } - - return false; - } - - private class GroupedByCluster - { - public IEphemeralCluster Cluster { get; set; } - public ITestCollection Collection { get; set; } - public List TestCases { get; set; } - } } } diff --git a/src/Elastic.Elasticsearch.Xunit/Sdk/TestCollectionRunner.cs b/src/Elastic.Elasticsearch.Xunit/Sdk/TestCollectionRunner.cs deleted file mode 100644 index 94083cb..0000000 --- a/src/Elastic.Elasticsearch.Xunit/Sdk/TestCollectionRunner.cs +++ /dev/null @@ -1,48 +0,0 @@ -// Licensed to Elasticsearch B.V under one or more agreements. -// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. -// See the LICENSE file in the project root for more information - -using System; -using System.Collections.Generic; -using System.Threading; -using System.Threading.Tasks; -using Xunit.Abstractions; -using Xunit.Sdk; - -namespace Elastic.Elasticsearch.Xunit.Sdk -{ - internal class TestCollectionRunner : XunitTestCollectionRunner - { - private readonly Dictionary _assemblyFixtureMappings; - private readonly IMessageSink _diagnosticMessageSink; - - public TestCollectionRunner(Dictionary assemblyFixtureMappings, - ITestCollection testCollection, - IEnumerable testCases, - IMessageSink diagnosticMessageSink, - IMessageBus messageBus, - ITestCaseOrderer testCaseOrderer, - ExceptionAggregator aggregator, - CancellationTokenSource cancellationTokenSource) - : base(testCollection, testCases, diagnosticMessageSink, messageBus, testCaseOrderer, aggregator, - cancellationTokenSource) - { - _assemblyFixtureMappings = assemblyFixtureMappings; - _diagnosticMessageSink = diagnosticMessageSink; - } - - protected override Task RunTestClassAsync(ITestClass testClass, IReflectionTypeInfo @class, - IEnumerable testCases) - { - // whats this doing exactly?? - var combinedFixtures = new Dictionary(_assemblyFixtureMappings); - foreach (var kvp in CollectionFixtureMappings) - combinedFixtures[kvp.Key] = kvp.Value; - - // We've done everything we need, so hand back off to default Xunit implementation for class runner - return new XunitTestClassRunner(testClass, @class, testCases, _diagnosticMessageSink, MessageBus, - TestCaseOrderer, new ExceptionAggregator(Aggregator), CancellationTokenSource, combinedFixtures) - .RunAsync(); - } - } -} diff --git a/src/Elastic.Elasticsearch.Xunit/Sdk/TestFrameworkExecutor.cs b/src/Elastic.Elasticsearch.Xunit/Sdk/TestFrameworkExecutor.cs deleted file mode 100644 index ab2fcea..0000000 --- a/src/Elastic.Elasticsearch.Xunit/Sdk/TestFrameworkExecutor.cs +++ /dev/null @@ -1,91 +0,0 @@ -// Licensed to Elasticsearch B.V under one or more agreements. -// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. -// See the LICENSE file in the project root for more information - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Reflection; -using Elastic.Elasticsearch.Managed; -using Xunit.Abstractions; -using Xunit.Sdk; - -namespace Elastic.Elasticsearch.Xunit.Sdk -{ - internal class TestFrameworkExecutor : XunitTestFrameworkExecutor - { - public TestFrameworkExecutor(AssemblyName a, ISourceInformationProvider sip, IMessageSink d) : base(a, sip, d) - { - } - - public ElasticXunitRunOptions Options { get; set; } - - public override void RunAll(IMessageSink executionMessageSink, ITestFrameworkDiscoveryOptions discoveryOptions, - ITestFrameworkExecutionOptions executionOptions) - { - discoveryOptions.SetValue(nameof(ElasticXunitRunOptions.Version), Options.Version); - discoveryOptions.SetValue(nameof(ElasticXunitRunOptions.RunIntegrationTests), Options.RunIntegrationTests); - discoveryOptions.SetValue(nameof(ElasticXunitRunOptions.IntegrationTestsMayUseAlreadyRunningNode), - Options.IntegrationTestsMayUseAlreadyRunningNode); - discoveryOptions.SetValue(nameof(ElasticXunitRunOptions.RunUnitTests), Options.RunUnitTests); - discoveryOptions.SetValue(nameof(ElasticXunitRunOptions.TestFilter), Options.TestFilter); - discoveryOptions.SetValue(nameof(ElasticXunitRunOptions.ClusterFilter), Options.ClusterFilter); - - executionOptions.SetValue(nameof(ElasticXunitRunOptions.Version), Options.Version); - executionOptions.SetValue(nameof(ElasticXunitRunOptions.RunIntegrationTests), Options.RunIntegrationTests); - executionOptions.SetValue(nameof(ElasticXunitRunOptions.IntegrationTestsMayUseAlreadyRunningNode), - Options.IntegrationTestsMayUseAlreadyRunningNode); - executionOptions.SetValue(nameof(ElasticXunitRunOptions.RunUnitTests), Options.RunUnitTests); - executionOptions.SetValue(nameof(ElasticXunitRunOptions.TestFilter), Options.TestFilter); - executionOptions.SetValue(nameof(ElasticXunitRunOptions.ClusterFilter), Options.ClusterFilter); - - base.RunAll(executionMessageSink, discoveryOptions, executionOptions); - } - - - public override void RunTests(IEnumerable testCases, IMessageSink executionMessageSink, - ITestFrameworkExecutionOptions executionOptions) - { - executionOptions.SetValue(nameof(ElasticXunitRunOptions.Version), Options.Version); - executionOptions.SetValue(nameof(ElasticXunitRunOptions.RunIntegrationTests), Options.RunIntegrationTests); - executionOptions.SetValue(nameof(ElasticXunitRunOptions.IntegrationTestsMayUseAlreadyRunningNode), - Options.IntegrationTestsMayUseAlreadyRunningNode); - executionOptions.SetValue(nameof(ElasticXunitRunOptions.RunUnitTests), Options.RunUnitTests); - executionOptions.SetValue(nameof(ElasticXunitRunOptions.TestFilter), Options.TestFilter); - executionOptions.SetValue(nameof(ElasticXunitRunOptions.ClusterFilter), Options.ClusterFilter); - base.RunTests(testCases, executionMessageSink, executionOptions); - } - - protected override async void RunTestCases(IEnumerable testCases, IMessageSink sink, - ITestFrameworkExecutionOptions options) - { - options.SetValue(nameof(ElasticXunitRunOptions.Version), Options.Version); - options.SetValue(nameof(ElasticXunitRunOptions.RunIntegrationTests), Options.RunIntegrationTests); - options.SetValue(nameof(ElasticXunitRunOptions.IntegrationTestsMayUseAlreadyRunningNode), - Options.IntegrationTestsMayUseAlreadyRunningNode); - options.SetValue(nameof(ElasticXunitRunOptions.RunUnitTests), Options.RunUnitTests); - options.SetValue(nameof(ElasticXunitRunOptions.TestFilter), Options.TestFilter); - options.SetValue(nameof(ElasticXunitRunOptions.ClusterFilter), Options.ClusterFilter); - try - { - using (var runner = - new TestAssemblyRunner(TestAssembly, testCases, DiagnosticMessageSink, sink, options)) - { - Options.OnBeforeTestsRun(); - await runner.RunAsync().ConfigureAwait(false); - Options.OnTestsFinished(runner.ClusterTotals, runner.FailedCollections); - } - } - catch (Exception e) - { - if (e is ElasticsearchCleanExitException || e is AggregateException ae && - ae.Flatten().InnerException is ElasticsearchCleanExitException) - sink.OnMessage(new TestAssemblyCleanupFailure(Enumerable.Empty(), TestAssembly, - new ElasticsearchCleanExitException("Node failed to start", e))); - else - sink.OnMessage(new TestAssemblyCleanupFailure(Enumerable.Empty(), TestAssembly, e)); - throw; - } - } - } -} diff --git a/src/Elastic.Elasticsearch.Xunit/XunitPlumbing/ElasticTestCaseDiscoverer.cs b/src/Elastic.Elasticsearch.Xunit/XunitPlumbing/ElasticTestCaseDiscoverer.cs index d259000..406795b 100644 --- a/src/Elastic.Elasticsearch.Xunit/XunitPlumbing/ElasticTestCaseDiscoverer.cs +++ b/src/Elastic.Elasticsearch.Xunit/XunitPlumbing/ElasticTestCaseDiscoverer.cs @@ -22,8 +22,11 @@ protected ElasticTestCaseDiscoverer(IMessageSink diagnosticMessageSink) => DiagnosticMessageSink = diagnosticMessageSink; /// - public IEnumerable Discover(ITestFrameworkDiscoveryOptions discoveryOptions, - ITestMethod testMethod, IAttributeInfo factAttribute) => + public IEnumerable Discover( + ITestFrameworkDiscoveryOptions discoveryOptions, + ITestMethod testMethod, + IAttributeInfo factAttribute + ) => SkipMethod(discoveryOptions, testMethod, out var skipReason) ? string.IsNullOrEmpty(skipReason) ? new IXunitTestCase[] { } @@ -47,17 +50,6 @@ protected virtual bool SkipMethod(ITestFrameworkDiscoveryOptions discoveryOption return false; } - protected static TValue GetAttribute(ITestMethod testMethod, string propertyName) - where TAttribute : Attribute - { - var classAttributes = testMethod.TestClass.Class.GetCustomAttributes(typeof(TAttribute)) ?? - Enumerable.Empty(); - var methodAttributes = testMethod.Method.GetCustomAttributes(typeof(TAttribute)) ?? - Enumerable.Empty(); - var attribute = classAttributes.Concat(methodAttributes).FirstOrDefault(); - return attribute == null ? default(TValue) : attribute.GetNamedArgument(propertyName); - } - protected static IList GetAttributes(ITestMethod testMethod) where TAttribute : Attribute { @@ -67,16 +59,5 @@ protected static IList GetAttributes(ITestMethod tes Enumerable.Empty(); return classAttributes.Concat(methodAttributes).ToList(); } - - protected static IEnumerable GetAttributes(ITestMethod testMethod, - string propertyName) - where TAttribute : Attribute - { - var classAttributes = testMethod.TestClass.Class.GetCustomAttributes(typeof(TAttribute)); - var methodAttributes = testMethod.Method.GetCustomAttributes(typeof(TAttribute)); - return classAttributes - .Concat(methodAttributes) - .Select(a => a.GetNamedArgument(propertyName)); - } } } diff --git a/src/Elastic.Elasticsearch.Xunit/XunitPlumbing/IntegrationTestClusterAttribute.cs b/src/Elastic.Elasticsearch.Xunit/XunitPlumbing/IntegrationTestClusterAttribute.cs deleted file mode 100644 index 1ff468a..0000000 --- a/src/Elastic.Elasticsearch.Xunit/XunitPlumbing/IntegrationTestClusterAttribute.cs +++ /dev/null @@ -1,23 +0,0 @@ -// Licensed to Elasticsearch B.V under one or more agreements. -// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. -// See the LICENSE file in the project root for more information - -using System; -using Elastic.Elasticsearch.Xunit.Sdk; - -namespace Elastic.Elasticsearch.Xunit.XunitPlumbing -{ - [AttributeUsage(AttributeTargets.Class, AllowMultiple = false)] - public class IntegrationTestClusterAttribute : Attribute - { - public IntegrationTestClusterAttribute(Type clusterType) - { - if (!TestAssemblyRunner.IsAnIntegrationTestClusterType(clusterType)) - throw new ArgumentException( - $"Cluster must be subclass of {nameof(XunitClusterBase)} or {nameof(XunitClusterBase)}<>"); - ClusterType = clusterType; - } - - public Type ClusterType { get; } - } -} diff --git a/src/Elastic.Elasticsearch.Xunit/XunitPlumbing/IntegrationTestDiscoverer.cs b/src/Elastic.Elasticsearch.Xunit/XunitPlumbing/IntegrationTestDiscoverer.cs index 69cbad4..e347be2 100644 --- a/src/Elastic.Elasticsearch.Xunit/XunitPlumbing/IntegrationTestDiscoverer.cs +++ b/src/Elastic.Elasticsearch.Xunit/XunitPlumbing/IntegrationTestDiscoverer.cs @@ -34,11 +34,11 @@ public abstract class SkipTestAttributeBase : Attribute /// /// An Xunit integration test /// - [XunitTestCaseDiscoverer("Elastic.Elasticsearch.Xunit.XunitPlumbing.IntegrationTestDiscoverer", - "Elastic.Elasticsearch.Xunit")] - public class I : FactAttribute - { - } + [XunitTestCaseDiscoverer( + "Elastic.Elasticsearch.Xunit.XunitPlumbing.IntegrationTestDiscoverer", + "Elastic.Elasticsearch.Xunit" + )] + public class I : FactAttribute { } /// /// A test discoverer used to discover integration tests cases attached @@ -51,19 +51,22 @@ public IntegrationTestDiscoverer(IMessageSink diagnosticMessageSink) : base(diag } /// - protected override bool SkipMethod(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, - out string skipReason) + protected override bool SkipMethod( + ITestFrameworkDiscoveryOptions discoveryOptions, + ITestMethod testMethod, + out string skipReason + ) { skipReason = null; var runIntegrationTests = discoveryOptions.GetValue(nameof(ElasticXunitRunOptions.RunIntegrationTests)); if (!runIntegrationTests) return true; - var cluster = TestAssemblyRunner.GetClusterForClass(testMethod.TestClass.Class); + var cluster = TestAssemblyRunner.GetClusterFixtureType(testMethod.TestClass.Class); if (cluster == null) { skipReason += - $"{testMethod.TestClass.Class.Name} does not define a cluster through IClusterFixture or {nameof(IntegrationTestClusterAttribute)}"; + $"{testMethod.TestClass.Class.Name} does not define a cluster through IClusterFixture"; return true; } @@ -71,7 +74,7 @@ protected override bool SkipMethod(ITestFrameworkDiscoveryOptions discoveryOptio discoveryOptions.GetValue(nameof(ElasticXunitRunOptions.Version)); // Skip if the version we are testing against is attributed to be skipped do not run the test nameof(SkipVersionAttribute.Ranges) - var skipVersionAttribute = Enumerable.FirstOrDefault(GetAttributes(testMethod)); + var skipVersionAttribute = GetAttributes(testMethod).FirstOrDefault(); if (skipVersionAttribute != null) { var skipVersionRanges = diff --git a/src/Elastic.Elasticsearch.Xunit/XunitPlumbing/UnitTestDiscoverer.cs b/src/Elastic.Elasticsearch.Xunit/XunitPlumbing/UnitTestDiscoverer.cs index 1f27ae0..954db69 100644 --- a/src/Elastic.Elasticsearch.Xunit/XunitPlumbing/UnitTestDiscoverer.cs +++ b/src/Elastic.Elasticsearch.Xunit/XunitPlumbing/UnitTestDiscoverer.cs @@ -12,8 +12,10 @@ namespace Elastic.Elasticsearch.Xunit.XunitPlumbing /// /// An Xunit unit test /// - [XunitTestCaseDiscoverer("Elastic.Elasticsearch.Xunit.XunitPlumbing.UnitTestDiscoverer", - "Elastic.Elasticsearch.Xunit")] + [XunitTestCaseDiscoverer( + "Elastic.Elasticsearch.Xunit.XunitPlumbing.UnitTestDiscoverer", + "Elastic.Elasticsearch.Xunit" + )] public class U : FactAttribute { } @@ -29,7 +31,9 @@ public UnitTestDiscoverer(IMessageSink diagnosticMessageSink) : base(diagnosticM } /// - protected override bool SkipMethod(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, + protected override bool SkipMethod( + ITestFrameworkDiscoveryOptions discoveryOptions, + ITestMethod testMethod, out string skipReason) { skipReason = null; diff --git a/src/Elastic.Xunit/Elastic.Xunit.csproj b/src/Elastic.Xunit/Elastic.Xunit.csproj new file mode 100644 index 0000000..4db8923 --- /dev/null +++ b/src/Elastic.Xunit/Elastic.Xunit.csproj @@ -0,0 +1,14 @@ + + + + netstandard2.0;netstandard2.1;net462 + enable + enable + false + + + + + + + diff --git a/src/Elastic.Xunit/ForEachAsyncExtensions.cs b/src/Elastic.Xunit/ForEachAsyncExtensions.cs new file mode 100644 index 0000000..6224bc7 --- /dev/null +++ b/src/Elastic.Xunit/ForEachAsyncExtensions.cs @@ -0,0 +1,24 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; + +namespace Elastic.Xunit; + +internal static class ForEachAsyncExtensions +{ + internal static Task ForEachAsync(this IEnumerable source, int dop, Func body) => + Task.WhenAll( + from partition in Partitioner.Create(source).GetPartitions(dop) + select Task.Run(async delegate + { + using (partition) + while (partition.MoveNext()) + await body(partition.Current).ConfigureAwait(false); + })); +} diff --git a/src/Elastic.Xunit/PartitioningConfigurationAttribute.cs b/src/Elastic.Xunit/PartitioningConfigurationAttribute.cs new file mode 100644 index 0000000..96cdd4e --- /dev/null +++ b/src/Elastic.Xunit/PartitioningConfigurationAttribute.cs @@ -0,0 +1,46 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System; +using System.Linq; +using System.Reflection; +using System.Runtime.ExceptionServices; + +namespace Elastic.Xunit; + +/// +/// An assembly attribute that specifies the +/// for Xunit tests within the assembly. +/// +[AttributeUsage(AttributeTargets.Assembly)] +public class PartitioningConfigurationAttribute : Attribute +{ + private readonly Type _type; + + /// + /// Creates a new instance of + /// + /// + /// A type deriving from that specifies the run options + /// + public PartitioningConfigurationAttribute(Type type) => _type = type; + + private TOptions? GetOptions() where TOptions : PartitioningRunOptions, new() + { + var options = Activator.CreateInstance(_type) as TOptions; + return options ?? new TOptions(); + } + + public static TOptions GetOptions(Assembly assembly) where TOptions : PartitioningRunOptions, new() + { + var options = assembly + .GetCustomAttributes() + .OfType() + .FirstOrDefault() + ?.GetOptions() + ?? new TOptions(); + + return options; + } +} diff --git a/src/Elastic.Xunit/PartitioningRunOptions.cs b/src/Elastic.Xunit/PartitioningRunOptions.cs new file mode 100644 index 0000000..8dda5ca --- /dev/null +++ b/src/Elastic.Xunit/PartitioningRunOptions.cs @@ -0,0 +1,63 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Diagnostics; +using Xunit.Abstractions; + +namespace Elastic.Xunit; + +/// +/// The Xunit test runner options +/// +public class PartitioningRunOptions : IRunOptions +{ + /// + /// A global test filter that can be used to only run certain tests. + /// Accepts a comma separated list of filters + /// + public string? TestFilter { get; set; } + + public string? GroupFilter { get; set; } + + /// + /// Called when the tests have finished running successfully + /// + /// Per cluster timings of the total test time, including starting Elasticsearch + /// All collection of failed cluster, failed tests tuples + public virtual void OnTestsFinished( + Dictionary? runnerClusterTotals, + ConcurrentBag>? runnerFailedCollections) + { + } + + /// + /// Called before tests run. An ideal place to perform actions such as writing information to + /// . + /// + public virtual void OnBeforeTestsRun() + { + } + + public virtual void SetOptions(ITestFrameworkDiscoveryOptions discoveryOptions) + { + discoveryOptions.SetValue(nameof(GroupFilter), GroupFilter); + discoveryOptions.SetValue(nameof(TestFilter), TestFilter); + + } + + public virtual void SetOptions(ITestFrameworkExecutionOptions executionOptions) + { + executionOptions.SetValue(nameof(GroupFilter), GroupFilter); + executionOptions.SetValue(nameof(GroupFilter), TestFilter); + + } +} + +public interface IRunOptions +{ + public void SetOptions(ITestFrameworkDiscoveryOptions discoveryOptions); +} diff --git a/src/Elastic.Xunit/PartitioningTestAssemblyRunner.cs b/src/Elastic.Xunit/PartitioningTestAssemblyRunner.cs new file mode 100644 index 0000000..c9bbbbe --- /dev/null +++ b/src/Elastic.Xunit/PartitioningTestAssemblyRunner.cs @@ -0,0 +1,327 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Xunit; +using Xunit.Abstractions; +using Xunit.Sdk; + +namespace Elastic.Xunit +{ + public interface ITestAssemblyRunnerFactory + { + public XunitTestAssemblyRunner Create( + ITestAssembly testAssembly, + IEnumerable testCases, + IMessageSink diagnosticMessageSink, + IMessageSink executionMessageSink, + ITestFrameworkExecutionOptions executionOptions + ); + } + + public class PartitioningTestRunnerFactory : ITestAssemblyRunnerFactory + { + public XunitTestAssemblyRunner Create(ITestAssembly testAssembly, IEnumerable testCases, + IMessageSink diagnosticMessageSink, + IMessageSink executionMessageSink, ITestFrameworkExecutionOptions executionOptions) => + new PartitioningTestAssemblyRunner(testAssembly, testCases, diagnosticMessageSink, executionMessageSink, + executionOptions); + } + + public interface ITestPartition : IAsyncLifetime + { + public int? MaxConcurrency { get; } + } + + // ReSharper disable once UnusedTypeParameter + public interface IPartitionFixture where TPartition : ITestPartition + { + } + + public class PartitioningTestAssemblyRunner : PartitioningTestAssemblyRunner + { + public PartitioningTestAssemblyRunner( + ITestAssembly testAssembly, + IEnumerable testCases, + IMessageSink diagnosticMessageSink, + IMessageSink executionMessageSink, + ITestFrameworkExecutionOptions executionOptions) + : base(testAssembly, testCases, diagnosticMessageSink, executionMessageSink, executionOptions, + typeof(IPartitionFixture<>)) + { + } + + protected override async Task UseStateAndRun(ITestPartition partition, Func runGroup) + { + await using (partition) + { + await partition.InitializeAsync().ConfigureAwait(false); + await runGroup(partition.MaxConcurrency).ConfigureAwait(false); + } + } + } + + public record TestType + { + public Type? Type { get; set; } + } + + public class TestPartitionGrouping + { + public Type? Type { get; set; } + public ITestCollection Collection { get; set; } = null!; + public List TestCases { get; set; } = null!; + } + + public abstract class PartitioningTestAssemblyRunner : XunitTestAssemblyRunner + where TState : class + { + private readonly Type _fixtureType; + private readonly Dictionary _assemblyFixtureMappings = new(); + + protected Dictionary> Partionings { get; } + + protected PartitioningTestAssemblyRunner(ITestAssembly testAssembly, + IEnumerable testCases, + IMessageSink diagnosticMessageSink, + IMessageSink executionMessageSink, + ITestFrameworkExecutionOptions executionOptions, Type fixtureType) + : base(testAssembly, testCases, diagnosticMessageSink, executionMessageSink, executionOptions) + { + _fixtureType = fixtureType; + GroupFilter = executionOptions.GetValue(nameof(PartitioningRunOptions.GroupFilter)); + TestFilter = executionOptions.GetValue(nameof(PartitioningRunOptions.TestFilter)); + + var testCollections = OrderTestCollections(); + + var cases = + from testCollection in testCollections + from classes in testCollection.Item2 + .Select(collection => collection.TestMethod.TestClass.Class) + .Distinct() + let partition = GetPartitionFixtureType(classes) + let testcase = new TestPartitionGrouping + { + Collection = testCollection.Item1, TestCases = testCollection.Item2, Type = partition + } + select testcase; + + Partionings = cases + .GroupBy(c => c.Type) + .OrderBy(g => g.Count()) + .ToDictionary(k => new TestType { Type = k.Key }, v => v.Select(g => g)); + + // types need to be instantiated ahead of time in order for xunit constructor injection checks. + foreach (var partitioning in Partionings) + { + var partitionType = partitioning.Key.Type; + if (partitionType == null) + continue; + + var state = CreatePartitionStateInstance(partitionType); + if (state != null) + _assemblyFixtureMappings[partitionType] = state; + } + } + + //threading guess + private static int DefaultConcurrency => Environment.ProcessorCount * 4; + + public ConcurrentBag Summaries { get; } = new(); + + public ConcurrentBag> FailedCollections { get; } = new(); + + public Dictionary ClusterTotals { get; } = new(); + + private string GroupFilter { get; } + private string TestFilter { get; } + + protected override Task RunTestCollectionAsync( + IMessageBus b, + ITestCollection c, + IEnumerable t, CancellationTokenSource s) + { + var aggregator = new ExceptionAggregator(Aggregator); + var fixtureObjects = new Dictionary(); + foreach (var kv in _assemblyFixtureMappings) + fixtureObjects.Add(kv.Key, kv.Value); + return new TestCollectionRunner(fixtureObjects, c, t, DiagnosticMessageSink, b, TestCaseOrderer, aggregator, + s) + .RunAsync(); + } + + protected override async Task RunTestCollectionsAsync(IMessageBus messageBus, + CancellationTokenSource cancellationTokenSource) => + await RunAllTests(messageBus, cancellationTokenSource) + .ConfigureAwait(false); + + protected async Task RunGroupedTestCollections( + IEnumerable source, + int? concurrency, + IMessageBus messageBus, string[] testFilters, CancellationTokenSource ctx) => + await source + .ForEachAsync(Math.Max(concurrency ?? 0, DefaultConcurrency), + async g => + { + await RunTestCollections(messageBus, ctx, g, testFilters).ConfigureAwait(false); + }) + .ConfigureAwait(false); + + + protected abstract Task UseStateAndRun(TState state, Func runGroup); + + protected async Task RunAllWithoutPartitionFixture( + IMessageBus messageBus, + CancellationTokenSource ctx + ) => + await RunWithoutPartitionFixture(Partionings.SelectMany(g => g.Value), messageBus, ctx) + .ConfigureAwait(false); + + protected async Task RunWithoutPartitionFixture( + IEnumerable partitionTests, + IMessageBus messageBus, CancellationTokenSource ctx) + { + var testFilters = CreateTestFilters(TestFilter); + + await RunGroupedTestCollections( + partitionTests, DefaultConcurrency, messageBus, + testFilters, ctx) + .ConfigureAwait(false); + + return new RunSummary + { + Total = Summaries.Sum(s => s.Total), + Failed = Summaries.Sum(s => s.Failed), + Skipped = Summaries.Sum(s => s.Skipped) + }; + } + + protected async Task RunAllTests(IMessageBus messageBus, CancellationTokenSource ctx) + { + var testFilters = CreateTestFilters(TestFilter); + foreach (var partitioning in Partionings) + { + var partitionType = partitioning.Key.Type; + if (partitionType == null) + { + var summary =await RunWithoutPartitionFixture(partitioning.Value, messageBus, ctx).ConfigureAwait(false); + Summaries.Add(summary); + continue; + } + + var state = CreatePartitionStateInstance(partitionType); + if (state == null) + { + var testClass = partitioning.Value.Select(g => g.Collection.DisplayName).FirstOrDefault(); + throw new Exception($"{typeof(TState)} did not yield partition state for e.g: {testClass}"); + } + + var type = state.GetType(); + var partitionName = type.Name; + if (!MatchesGroupFilter(partitionName)) continue; + + var skipReasons = partitioning.Value.SelectMany(g => g.TestCases.Select(t => t.SkipReason)).ToList(); + var allSkipped = skipReasons.All(r => !string.IsNullOrWhiteSpace(r)); + if (allSkipped) + { + Summaries.Add(new RunSummary { Total = skipReasons.Count, Skipped = skipReasons.Count }); + continue; + } + + ClusterTotals.Add(partitionName, Stopwatch.StartNew()); + + await UseStateAndRun(state, async (concurrency) => + { + await RunGroupedTestCollections(partitioning.Value, concurrency, messageBus, testFilters, ctx) + .ConfigureAwait(false); + }).ConfigureAwait(false); + + + ClusterTotals[partitionName].Stop(); + } + + return new RunSummary + { + Total = Summaries.Sum(s => s.Total), + Failed = Summaries.Sum(s => s.Failed), + Skipped = Summaries.Sum(s => s.Skipped) + }; + } + + private async Task RunTestCollections( + IMessageBus messageBus, + CancellationTokenSource ctx, + TestPartitionGrouping g, + string[] testFilters + ) + { + var test = g.Collection.DisplayName.Replace("Test collection for", string.Empty).Trim(); + if (!MatchesATestFilter(test, testFilters)) return; + if (testFilters.Length > 0) Console.WriteLine(" -> " + test); + + try + { + var summary = await RunTestCollectionAsync(messageBus, g.Collection, g.TestCases, ctx) + .ConfigureAwait(false); + var type = g.Type; + var partitionName = type?.Name ?? "UNKNOWN"; + if (summary.Failed > 0) + FailedCollections.Add(Tuple.Create(partitionName, test)); + Summaries.Add(summary); + } + catch (TaskCanceledException) + { + // TODO: What should happen here? + } + } + + private static string[] CreateTestFilters(string testFilters) => + testFilters?.Split(',').Select(s => s.Trim()).Where(s => !string.IsNullOrWhiteSpace(s)).ToArray() + ?? new string[] { }; + + private static bool MatchesATestFilter(string test, IReadOnlyCollection testFilters) + { + if (testFilters.Count == 0 || string.IsNullOrWhiteSpace(test)) return true; + return testFilters + .Any(filter => test.IndexOf(filter, StringComparison.OrdinalIgnoreCase) >= 0); + } + + private bool MatchesGroupFilter(string cluster) + { + if (string.IsNullOrWhiteSpace(cluster) || string.IsNullOrWhiteSpace(GroupFilter)) return true; + return GroupFilter + .Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries) + .Select(c => c.Trim()) + .Any(c => cluster.IndexOf(c, StringComparison.OrdinalIgnoreCase) >= 0); + } + + private TState? CreatePartitionStateInstance(Type partitionType) + { + if (_assemblyFixtureMappings.TryGetValue(partitionType, out var partition)) return partition; + Aggregator.Run(() => + { + var o = Activator.CreateInstance(partitionType); + partition = o as TState; + }); + _assemblyFixtureMappings.Add(partitionType, partition); + return partition; + } + + protected Type? GetPartitionFixtureType(ITypeInfo testClass) => + GetPartitionFixtureType(testClass, _fixtureType); + + protected static Type? GetPartitionFixtureType(ITypeInfo testClass, Type openGenericState) => + testClass.ToRuntimeType().GetInterfaces() + .Where(i => i.IsGenericType) + .Where(t => t.GetGenericTypeDefinition() == openGenericState) + .SelectMany(t => t.GetGenericArguments(), (_, a) => a) + .FirstOrDefault(); + } +} diff --git a/src/Elastic.Xunit/PartitioningTestFramework.cs b/src/Elastic.Xunit/PartitioningTestFramework.cs new file mode 100644 index 0000000..ff2ebd6 --- /dev/null +++ b/src/Elastic.Xunit/PartitioningTestFramework.cs @@ -0,0 +1,27 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.Reflection; +using Xunit.Abstractions; +using Xunit.Sdk; + +namespace Elastic.Xunit; + +public class PartitioningTestFramework : XunitTestFramework + where TOptions : PartitioningRunOptions, new() + where TRunnerFactory : ITestAssemblyRunnerFactory, new() +{ + public PartitioningTestFramework(IMessageSink messageSink) : base(messageSink) { } + + protected override ITestFrameworkDiscoverer CreateDiscoverer(IAssemblyInfo assemblyInfo) => + new PartitioningTestFrameworkDiscoverer(assemblyInfo, SourceInformationProvider, DiagnosticMessageSink); + + protected override ITestFrameworkExecutor CreateExecutor(AssemblyName assemblyName) + { + var assembly = Assembly.Load(assemblyName); + var options = PartitioningConfigurationAttribute.GetOptions(assembly); + + return new PartitioningTestFrameworkExecutor(options, assemblyName, SourceInformationProvider, DiagnosticMessageSink); + } +} diff --git a/src/Elastic.Xunit/PartitioningTestFrameworkDiscoverer.cs b/src/Elastic.Xunit/PartitioningTestFrameworkDiscoverer.cs new file mode 100644 index 0000000..9c7b92a --- /dev/null +++ b/src/Elastic.Xunit/PartitioningTestFrameworkDiscoverer.cs @@ -0,0 +1,33 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.Reflection; +using Xunit.Abstractions; +using Xunit.Sdk; + +namespace Elastic.Xunit; + +internal class PartitioningTestFrameworkDiscoverer : XunitTestFrameworkDiscoverer + where TOptions : PartitioningRunOptions, new() +{ + public PartitioningTestFrameworkDiscoverer( + IAssemblyInfo assemblyInfo, + ISourceInformationProvider sourceProvider, + IMessageSink diagnosticMessageSink, + IXunitTestCollectionFactory? collectionFactory = null) : base( + assemblyInfo, sourceProvider, diagnosticMessageSink, collectionFactory) + { + var a = Assembly.Load(new AssemblyName(assemblyInfo.Name)); + Options = PartitioningConfigurationAttribute.GetOptions(a); + } + + private TOptions Options { get; } + + protected override bool FindTestsForType(ITestClass testClass, bool includeSourceInformation, + IMessageBus messageBus, ITestFrameworkDiscoveryOptions discoveryOptions) + { + Options.SetOptions(discoveryOptions); + return base.FindTestsForType(testClass, includeSourceInformation, messageBus, discoveryOptions); + } +} diff --git a/src/Elastic.Xunit/PartitioningTestFrameworkExecutor.cs b/src/Elastic.Xunit/PartitioningTestFrameworkExecutor.cs new file mode 100644 index 0000000..868de0f --- /dev/null +++ b/src/Elastic.Xunit/PartitioningTestFrameworkExecutor.cs @@ -0,0 +1,80 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using Xunit.Abstractions; +using Xunit.Sdk; + +namespace Elastic.Xunit +{ + internal class PartitioningTestFrameworkExecutor : XunitTestFrameworkExecutor + where TOptions : PartitioningRunOptions, new() + where TRunnerFactory : ITestAssemblyRunnerFactory, new() + { + public PartitioningTestFrameworkExecutor( + TOptions options, + AssemblyName a, + ISourceInformationProvider sip, + IMessageSink d + ) + : base(a, sip, d) + { + Options = options; + RunnerFactory = new TRunnerFactory(); + } + + private ITestAssemblyRunnerFactory RunnerFactory { get; } + + private TOptions Options { get; } + + public override void RunAll( + IMessageSink executionMessageSink, + ITestFrameworkDiscoveryOptions discoveryOptions, + ITestFrameworkExecutionOptions executionOptions + ) + { + Options.SetOptions(discoveryOptions); + Options.SetOptions(executionOptions); + + base.RunAll(executionMessageSink, discoveryOptions, executionOptions); + } + + + public override void RunTests( + IEnumerable testCases, + IMessageSink executionMessageSink, + ITestFrameworkExecutionOptions executionOptions) + { + Options.SetOptions(executionOptions); + base.RunTests(testCases, executionMessageSink, executionOptions); + } + + protected override async void RunTestCases( + IEnumerable testCases, + IMessageSink sink, + ITestFrameworkExecutionOptions options) + { + Options.SetOptions(options); + try + { + using var runner = RunnerFactory.Create(TestAssembly, testCases, DiagnosticMessageSink, sink, options); + Options.OnBeforeTestsRun(); + await runner.RunAsync().ConfigureAwait(false); + var r = runner as PartitioningTestAssemblyRunner; + Options.OnTestsFinished( + r?.ClusterTotals, + r?.FailedCollections + ); + } + catch (Exception e) + { + sink.OnMessage(new TestAssemblyCleanupFailure(Enumerable.Empty(), TestAssembly, e)); + throw; + } + } + } +} diff --git a/src/Elastic.Xunit/TestCollectionRunner.cs b/src/Elastic.Xunit/TestCollectionRunner.cs new file mode 100644 index 0000000..5530d9d --- /dev/null +++ b/src/Elastic.Xunit/TestCollectionRunner.cs @@ -0,0 +1,47 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Xunit.Abstractions; +using Xunit.Sdk; + +namespace Elastic.Xunit; + +internal class TestCollectionRunner : XunitTestCollectionRunner +{ + private readonly Dictionary _assemblyFixtureMappings; + private readonly IMessageSink _diagnosticMessageSink; + + public TestCollectionRunner(Dictionary assemblyFixtureMappings, + ITestCollection testCollection, + IEnumerable testCases, + IMessageSink diagnosticMessageSink, + IMessageBus messageBus, + ITestCaseOrderer testCaseOrderer, + ExceptionAggregator aggregator, + CancellationTokenSource cancellationTokenSource) + : base(testCollection, testCases, diagnosticMessageSink, messageBus, testCaseOrderer, aggregator, + cancellationTokenSource) + { + _assemblyFixtureMappings = assemblyFixtureMappings; + _diagnosticMessageSink = diagnosticMessageSink; + } + + protected override Task RunTestClassAsync(ITestClass testClass, IReflectionTypeInfo @class, + IEnumerable testCases) + { + // this ensures xunit can constructor inject our partition types + var combinedFixtures = new Dictionary(_assemblyFixtureMappings); + foreach (var kvp in CollectionFixtureMappings) + combinedFixtures[kvp.Key] = kvp.Value; + + // We've done everything we need, so hand back off to default Xunit implementation for class runner + return new XunitTestClassRunner(testClass, @class, testCases, _diagnosticMessageSink, MessageBus, + TestCaseOrderer, new ExceptionAggregator(Aggregator), CancellationTokenSource, combinedFixtures) + .RunAsync(); + } +} From 383e67b5d123784d0f9309b8c6d4ef550366b6b8 Mon Sep 17 00:00:00 2001 From: Martijn Laarman Date: Thu, 8 Feb 2024 16:52:40 +0100 Subject: [PATCH 2/5] small cleanups --- .../Sdk/TestAssemblyRunner.cs | 6 +- .../PartitioningConfigurationAttribute.cs | 3 +- src/Elastic.Xunit/PartitioningRunOptions.cs | 16 ++-- .../PartitioningTestAssemblyRunner.cs | 79 ++++++++----------- .../PartitioningTestFramework.cs | 13 ++- .../PartitioningTestFrameworkExecutor.cs | 8 +- 6 files changed, 53 insertions(+), 72 deletions(-) diff --git a/src/Elastic.Elasticsearch.Xunit/Sdk/TestAssemblyRunner.cs b/src/Elastic.Elasticsearch.Xunit/Sdk/TestAssemblyRunner.cs index f910cc9..89f626d 100644 --- a/src/Elastic.Elasticsearch.Xunit/Sdk/TestAssemblyRunner.cs +++ b/src/Elastic.Elasticsearch.Xunit/Sdk/TestAssemblyRunner.cs @@ -49,14 +49,14 @@ public TestAssemblyRunner(ITestAssembly testAssembly, private bool IntegrationTestsMayUseAlreadyRunningNode { get; } private bool RunUnitTests { get; } - protected override async Task RunTestCollectionsAsync(IMessageBus messageBus, + protected override async Task RunTestCollectionsAsync(IMessageBus bus, CancellationTokenSource cancellationTokenSource) { if (RunUnitTests && !RunIntegrationTests) - return await RunAllWithoutPartitionFixture(messageBus, cancellationTokenSource) + return await RunAllWithoutPartitionFixture(bus, cancellationTokenSource) .ConfigureAwait(false); - return await RunAllTests(messageBus, cancellationTokenSource) + return await RunAllTests(bus, cancellationTokenSource) .ConfigureAwait(false); } diff --git a/src/Elastic.Xunit/PartitioningConfigurationAttribute.cs b/src/Elastic.Xunit/PartitioningConfigurationAttribute.cs index 96cdd4e..8a88da4 100644 --- a/src/Elastic.Xunit/PartitioningConfigurationAttribute.cs +++ b/src/Elastic.Xunit/PartitioningConfigurationAttribute.cs @@ -5,7 +5,6 @@ using System; using System.Linq; using System.Reflection; -using System.Runtime.ExceptionServices; namespace Elastic.Xunit; @@ -26,7 +25,7 @@ public class PartitioningConfigurationAttribute : Attribute /// public PartitioningConfigurationAttribute(Type type) => _type = type; - private TOptions? GetOptions() where TOptions : PartitioningRunOptions, new() + private TOptions GetOptions() where TOptions : PartitioningRunOptions, new() { var options = Activator.CreateInstance(_type) as TOptions; return options ?? new TOptions(); diff --git a/src/Elastic.Xunit/PartitioningRunOptions.cs b/src/Elastic.Xunit/PartitioningRunOptions.cs index 8dda5ca..022257d 100644 --- a/src/Elastic.Xunit/PartitioningRunOptions.cs +++ b/src/Elastic.Xunit/PartitioningRunOptions.cs @@ -13,7 +13,7 @@ namespace Elastic.Xunit; /// /// The Xunit test runner options /// -public class PartitioningRunOptions : IRunOptions +public class PartitioningRunOptions { /// /// A global test filter that can be used to only run certain tests. @@ -26,11 +26,11 @@ public class PartitioningRunOptions : IRunOptions /// /// Called when the tests have finished running successfully /// - /// Per cluster timings of the total test time, including starting Elasticsearch - /// All collection of failed cluster, failed tests tuples + /// Per cluster timings of the total test time, including starting Elasticsearch + /// All collection of failed cluster, failed tests tuples public virtual void OnTestsFinished( - Dictionary? runnerClusterTotals, - ConcurrentBag>? runnerFailedCollections) + Dictionary partitionTimings, + ConcurrentBag> failedPartitionTests) { } @@ -46,7 +46,6 @@ public virtual void SetOptions(ITestFrameworkDiscoveryOptions discoveryOptions) { discoveryOptions.SetValue(nameof(GroupFilter), GroupFilter); discoveryOptions.SetValue(nameof(TestFilter), TestFilter); - } public virtual void SetOptions(ITestFrameworkExecutionOptions executionOptions) @@ -56,8 +55,3 @@ public virtual void SetOptions(ITestFrameworkExecutionOptions executionOptions) } } - -public interface IRunOptions -{ - public void SetOptions(ITestFrameworkDiscoveryOptions discoveryOptions); -} diff --git a/src/Elastic.Xunit/PartitioningTestAssemblyRunner.cs b/src/Elastic.Xunit/PartitioningTestAssemblyRunner.cs index c9bbbbe..1c8d9c0 100644 --- a/src/Elastic.Xunit/PartitioningTestAssemblyRunner.cs +++ b/src/Elastic.Xunit/PartitioningTestAssemblyRunner.cs @@ -41,9 +41,7 @@ public interface ITestPartition : IAsyncLifetime } // ReSharper disable once UnusedTypeParameter - public interface IPartitionFixture where TPartition : ITestPartition - { - } + public interface IPartitionFixture where TPartition : ITestPartition { } public class PartitioningTestAssemblyRunner : PartitioningTestAssemblyRunner { @@ -84,7 +82,7 @@ public abstract class PartitioningTestAssemblyRunner : XunitTestAssembly where TState : class { private readonly Type _fixtureType; - private readonly Dictionary _assemblyFixtureMappings = new(); + private readonly Dictionary _partitionFixtureInstances = new(); protected Dictionary> Partionings { get; } @@ -127,22 +125,22 @@ from classes in testCollection.Item2 var state = CreatePartitionStateInstance(partitionType); if (state != null) - _assemblyFixtureMappings[partitionType] = state; + _partitionFixtureInstances[partitionType] = state; } } //threading guess private static int DefaultConcurrency => Environment.ProcessorCount * 4; - public ConcurrentBag Summaries { get; } = new(); - + private ConcurrentBag Summaries { get; } = new(); public ConcurrentBag> FailedCollections { get; } = new(); - public Dictionary ClusterTotals { get; } = new(); private string GroupFilter { get; } private string TestFilter { get; } + protected abstract Task UseStateAndRun(TState state, Func runGroup); + protected override Task RunTestCollectionAsync( IMessageBus b, ITestCollection c, @@ -150,39 +148,14 @@ protected override Task RunTestCollectionAsync( { var aggregator = new ExceptionAggregator(Aggregator); var fixtureObjects = new Dictionary(); - foreach (var kv in _assemblyFixtureMappings) + foreach (var kv in _partitionFixtureInstances) fixtureObjects.Add(kv.Key, kv.Value); - return new TestCollectionRunner(fixtureObjects, c, t, DiagnosticMessageSink, b, TestCaseOrderer, aggregator, - s) - .RunAsync(); + var runner = new TestCollectionRunner(fixtureObjects, c, t, DiagnosticMessageSink, b, TestCaseOrderer, aggregator, s); + return runner.RunAsync(); } - protected override async Task RunTestCollectionsAsync(IMessageBus messageBus, - CancellationTokenSource cancellationTokenSource) => - await RunAllTests(messageBus, cancellationTokenSource) - .ConfigureAwait(false); - - protected async Task RunGroupedTestCollections( - IEnumerable source, - int? concurrency, - IMessageBus messageBus, string[] testFilters, CancellationTokenSource ctx) => - await source - .ForEachAsync(Math.Max(concurrency ?? 0, DefaultConcurrency), - async g => - { - await RunTestCollections(messageBus, ctx, g, testFilters).ConfigureAwait(false); - }) - .ConfigureAwait(false); - - - protected abstract Task UseStateAndRun(TState state, Func runGroup); - - protected async Task RunAllWithoutPartitionFixture( - IMessageBus messageBus, - CancellationTokenSource ctx - ) => - await RunWithoutPartitionFixture(Partionings.SelectMany(g => g.Value), messageBus, ctx) - .ConfigureAwait(false); + protected async Task RunAllWithoutPartitionFixture(IMessageBus bus, CancellationTokenSource ctx) => + await RunWithoutPartitionFixture(Partionings.SelectMany(g => g.Value), bus, ctx).ConfigureAwait(false); protected async Task RunWithoutPartitionFixture( IEnumerable partitionTests, @@ -190,7 +163,7 @@ protected async Task RunWithoutPartitionFixture( { var testFilters = CreateTestFilters(TestFilter); - await RunGroupedTestCollections( + await RunPartitionGroupConcurrently( partitionTests, DefaultConcurrency, messageBus, testFilters, ctx) .ConfigureAwait(false); @@ -203,6 +176,9 @@ await RunGroupedTestCollections( }; } + protected override async Task RunTestCollectionsAsync(IMessageBus bus, CancellationTokenSource ctx) => + await RunAllTests(bus, ctx).ConfigureAwait(false); + protected async Task RunAllTests(IMessageBus messageBus, CancellationTokenSource ctx) { var testFilters = CreateTestFilters(TestFilter); @@ -239,11 +215,10 @@ protected async Task RunAllTests(IMessageBus messageBus, Cancellatio await UseStateAndRun(state, async (concurrency) => { - await RunGroupedTestCollections(partitioning.Value, concurrency, messageBus, testFilters, ctx) + await RunPartitionGroupConcurrently(partitioning.Value, concurrency, messageBus, testFilters, ctx) .ConfigureAwait(false); }).ConfigureAwait(false); - ClusterTotals[partitionName].Stop(); } @@ -255,7 +230,17 @@ await RunGroupedTestCollections(partitioning.Value, concurrency, messageBus, tes }; } - private async Task RunTestCollections( + private async Task RunPartitionGroupConcurrently( + IEnumerable source, + int? concurrency, + IMessageBus bus, string[] testFilters, CancellationTokenSource ctx) => + await source + .ForEachAsync(Math.Max(concurrency ?? 0, DefaultConcurrency), + async g => await ExecutePartitionGrouping(bus, ctx, g, testFilters).ConfigureAwait(false) + ) + .ConfigureAwait(false); + + private async Task ExecutePartitionGrouping( IMessageBus messageBus, CancellationTokenSource ctx, TestPartitionGrouping g, @@ -264,8 +249,6 @@ string[] testFilters { var test = g.Collection.DisplayName.Replace("Test collection for", string.Empty).Trim(); if (!MatchesATestFilter(test, testFilters)) return; - if (testFilters.Length > 0) Console.WriteLine(" -> " + test); - try { var summary = await RunTestCollectionAsync(messageBus, g.Collection, g.TestCases, ctx) @@ -282,7 +265,7 @@ string[] testFilters } } - private static string[] CreateTestFilters(string testFilters) => + private static string[] CreateTestFilters(string? testFilters) => testFilters?.Split(',').Select(s => s.Trim()).Where(s => !string.IsNullOrWhiteSpace(s)).ToArray() ?? new string[] { }; @@ -304,17 +287,17 @@ private bool MatchesGroupFilter(string cluster) private TState? CreatePartitionStateInstance(Type partitionType) { - if (_assemblyFixtureMappings.TryGetValue(partitionType, out var partition)) return partition; + if (_partitionFixtureInstances.TryGetValue(partitionType, out var partition)) return partition; Aggregator.Run(() => { var o = Activator.CreateInstance(partitionType); partition = o as TState; }); - _assemblyFixtureMappings.Add(partitionType, partition); + _partitionFixtureInstances.Add(partitionType, partition); return partition; } - protected Type? GetPartitionFixtureType(ITypeInfo testClass) => + private Type? GetPartitionFixtureType(ITypeInfo testClass) => GetPartitionFixtureType(testClass, _fixtureType); protected static Type? GetPartitionFixtureType(ITypeInfo testClass, Type openGenericState) => diff --git a/src/Elastic.Xunit/PartitioningTestFramework.cs b/src/Elastic.Xunit/PartitioningTestFramework.cs index ff2ebd6..5e9c762 100644 --- a/src/Elastic.Xunit/PartitioningTestFramework.cs +++ b/src/Elastic.Xunit/PartitioningTestFramework.cs @@ -8,11 +8,20 @@ namespace Elastic.Xunit; -public class PartitioningTestFramework : XunitTestFramework +// ReSharper disable once UnusedType.Global +/// +/// +/// +public class PartitioningTestFramework : PartitioningTestFramework +{ + public PartitioningTestFramework(IMessageSink messageSink) : base(messageSink) { } +} + +public abstract class PartitioningTestFramework : XunitTestFramework where TOptions : PartitioningRunOptions, new() where TRunnerFactory : ITestAssemblyRunnerFactory, new() { - public PartitioningTestFramework(IMessageSink messageSink) : base(messageSink) { } + protected PartitioningTestFramework(IMessageSink messageSink) : base(messageSink) { } protected override ITestFrameworkDiscoverer CreateDiscoverer(IAssemblyInfo assemblyInfo) => new PartitioningTestFrameworkDiscoverer(assemblyInfo, SourceInformationProvider, DiagnosticMessageSink); diff --git a/src/Elastic.Xunit/PartitioningTestFrameworkExecutor.cs b/src/Elastic.Xunit/PartitioningTestFrameworkExecutor.cs index 868de0f..10ec4ea 100644 --- a/src/Elastic.Xunit/PartitioningTestFrameworkExecutor.cs +++ b/src/Elastic.Xunit/PartitioningTestFrameworkExecutor.cs @@ -43,7 +43,6 @@ ITestFrameworkExecutionOptions executionOptions base.RunAll(executionMessageSink, discoveryOptions, executionOptions); } - public override void RunTests( IEnumerable testCases, IMessageSink executionMessageSink, @@ -64,11 +63,8 @@ protected override async void RunTestCases( using var runner = RunnerFactory.Create(TestAssembly, testCases, DiagnosticMessageSink, sink, options); Options.OnBeforeTestsRun(); await runner.RunAsync().ConfigureAwait(false); - var r = runner as PartitioningTestAssemblyRunner; - Options.OnTestsFinished( - r?.ClusterTotals, - r?.FailedCollections - ); + if (runner is PartitioningTestAssemblyRunner a) + Options.OnTestsFinished(a.ClusterTotals, a.FailedCollections); } catch (Exception e) { From 1dbf386a300322caefb03a02a0b32fcb7bc31f8d Mon Sep 17 00:00:00 2001 From: Martijn Laarman Date: Fri, 9 Feb 2024 14:35:14 +0100 Subject: [PATCH 3/5] Move to Nullean.Xunit.Partitions --- Elastic.Abstractions.sln | 7 - .../Elastic.Xunit.ExampleComplex/Setup.cs | 1 - .../Elastic.Elasticsearch.Xunit.csproj | 2 +- .../ElasticXunitConfigurationAttribute.cs | 4 +- .../ElasticXunitRunOptions.cs | 49 ++- .../Sdk/ElasticTestFramework.cs | 23 +- .../Sdk/TestAssemblyRunner.cs | 5 +- .../XunitPlumbing/SkippingTestCase.cs | 5 +- src/Elastic.Xunit/Elastic.Xunit.csproj | 14 - src/Elastic.Xunit/ForEachAsyncExtensions.cs | 24 -- .../PartitioningConfigurationAttribute.cs | 45 --- src/Elastic.Xunit/PartitioningRunOptions.cs | 57 ---- .../PartitioningTestAssemblyRunner.cs | 310 ------------------ .../PartitioningTestFramework.cs | 36 -- .../PartitioningTestFrameworkDiscoverer.cs | 33 -- .../PartitioningTestFrameworkExecutor.cs | 76 ----- src/Elastic.Xunit/TestCollectionRunner.cs | 47 --- 17 files changed, 67 insertions(+), 671 deletions(-) delete mode 100644 src/Elastic.Xunit/Elastic.Xunit.csproj delete mode 100644 src/Elastic.Xunit/ForEachAsyncExtensions.cs delete mode 100644 src/Elastic.Xunit/PartitioningConfigurationAttribute.cs delete mode 100644 src/Elastic.Xunit/PartitioningRunOptions.cs delete mode 100644 src/Elastic.Xunit/PartitioningTestAssemblyRunner.cs delete mode 100644 src/Elastic.Xunit/PartitioningTestFramework.cs delete mode 100644 src/Elastic.Xunit/PartitioningTestFrameworkDiscoverer.cs delete mode 100644 src/Elastic.Xunit/PartitioningTestFrameworkExecutor.cs delete mode 100644 src/Elastic.Xunit/TestCollectionRunner.cs diff --git a/Elastic.Abstractions.sln b/Elastic.Abstractions.sln index 412f115..9cada97 100644 --- a/Elastic.Abstractions.sln +++ b/Elastic.Abstractions.sln @@ -45,8 +45,6 @@ EndProjectSection EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Elastic.Ephemeral.Example", "examples\Elastic.Ephemeral.Example\Elastic.Ephemeral.Example.csproj", "{9666AFDC-B0E8-489C-A25A-17E67303A969}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Elastic.Xunit", "src\Elastic.Xunit\Elastic.Xunit.csproj", "{4D3CA43F-1653-4A6B-A5AC-FC3DB32F1DDA}" -EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -67,7 +65,6 @@ Global {C05F7B36-EEF7-4BCD-86A2-F5F1BB8CFEB9} = {77E78EDE-60D5-469A-B431-443A7966A243} {D6997ADC-E933-418E-831C-DE1A78897493} = {F75ACC18-D314-4F1F-88A3-2002EAC4E207} {9666AFDC-B0E8-489C-A25A-17E67303A969} = {9D154338-4AA8-40A9-A378-B27C05D45791} - {4D3CA43F-1653-4A6B-A5AC-FC3DB32F1DDA} = {77E78EDE-60D5-469A-B431-443A7966A243} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {AFADDCED-A7DD-43E7-B03C-27F57AC5C358} @@ -117,9 +114,5 @@ Global {9666AFDC-B0E8-489C-A25A-17E67303A969}.Debug|Any CPU.Build.0 = Debug|Any CPU {9666AFDC-B0E8-489C-A25A-17E67303A969}.Release|Any CPU.ActiveCfg = Release|Any CPU {9666AFDC-B0E8-489C-A25A-17E67303A969}.Release|Any CPU.Build.0 = Release|Any CPU - {4D3CA43F-1653-4A6B-A5AC-FC3DB32F1DDA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {4D3CA43F-1653-4A6B-A5AC-FC3DB32F1DDA}.Debug|Any CPU.Build.0 = Debug|Any CPU - {4D3CA43F-1653-4A6B-A5AC-FC3DB32F1DDA}.Release|Any CPU.ActiveCfg = Release|Any CPU - {4D3CA43F-1653-4A6B-A5AC-FC3DB32F1DDA}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection EndGlobal diff --git a/examples/Elastic.Xunit.ExampleComplex/Setup.cs b/examples/Elastic.Xunit.ExampleComplex/Setup.cs index 0fc7466..905289a 100644 --- a/examples/Elastic.Xunit.ExampleComplex/Setup.cs +++ b/examples/Elastic.Xunit.ExampleComplex/Setup.cs @@ -19,7 +19,6 @@ public class MyRunOptions : ElasticXunitRunOptions { public MyRunOptions() { - ClusterFilter = ""; RunUnitTests = true; RunIntegrationTests = true; IntegrationTestsMayUseAlreadyRunningNode = true; diff --git a/src/Elastic.Elasticsearch.Xunit/Elastic.Elasticsearch.Xunit.csproj b/src/Elastic.Elasticsearch.Xunit/Elastic.Elasticsearch.Xunit.csproj index 236f8b0..ccc07f1 100644 --- a/src/Elastic.Elasticsearch.Xunit/Elastic.Elasticsearch.Xunit.csproj +++ b/src/Elastic.Elasticsearch.Xunit/Elastic.Elasticsearch.Xunit.csproj @@ -6,10 +6,10 @@ elastic,elasticsearch,xunit,cluster,integration,test,ephemeral + - \ No newline at end of file diff --git a/src/Elastic.Elasticsearch.Xunit/ElasticXunitConfigurationAttribute.cs b/src/Elastic.Elasticsearch.Xunit/ElasticXunitConfigurationAttribute.cs index 37b0ff4..f7f280e 100644 --- a/src/Elastic.Elasticsearch.Xunit/ElasticXunitConfigurationAttribute.cs +++ b/src/Elastic.Elasticsearch.Xunit/ElasticXunitConfigurationAttribute.cs @@ -3,7 +3,7 @@ // See the LICENSE file in the project root for more information using System; -using Elastic.Xunit; +using Nullean.Xunit.Partitions; namespace Elastic.Elasticsearch.Xunit; @@ -12,7 +12,7 @@ namespace Elastic.Elasticsearch.Xunit; /// for Xunit tests within the assembly. /// [AttributeUsage(AttributeTargets.Assembly)] -public class ElasticXunitConfigurationAttribute : PartitioningConfigurationAttribute +public class ElasticXunitConfigurationAttribute : PartitionOptionsAttribute { /// Creates a new instance of . /// diff --git a/src/Elastic.Elasticsearch.Xunit/ElasticXunitRunOptions.cs b/src/Elastic.Elasticsearch.Xunit/ElasticXunitRunOptions.cs index b756bba..395aae4 100644 --- a/src/Elastic.Elasticsearch.Xunit/ElasticXunitRunOptions.cs +++ b/src/Elastic.Elasticsearch.Xunit/ElasticXunitRunOptions.cs @@ -3,20 +3,19 @@ // See the LICENSE file in the project root for more information using System; -using System.Collections.Concurrent; -using System.Collections.Generic; -using System.Diagnostics; +using System.Linq; using Elastic.Elasticsearch.Xunit.XunitPlumbing; using Elastic.Stack.ArtifactsApi; -using Elastic.Xunit; +using Nullean.Xunit.Partitions; using Xunit.Abstractions; +using static System.StringSplitOptions; namespace Elastic.Elasticsearch.Xunit { /// /// The Xunit test runner options /// - public class ElasticXunitRunOptions : PartitioningRunOptions + public class ElasticXunitRunOptions : PartitionOptions { /// /// Informs the runner whether we expect to run integration tests. Defaults to true @@ -40,7 +39,41 @@ public class ElasticXunitRunOptions : PartitioningRunOptions /// A global cluster filter that can be used to only run certain cluster's tests. /// Accepts a comma separated list of filters /// - public string ClusterFilter { get => GroupFilter; set => GroupFilter = value; } + [Obsolete("Use PartitionFilterRegex instead", false)] + public string ClusterFilter + { + get => PartitionFilterRegex; + set + { + if (string.IsNullOrWhiteSpace(value)) PartitionFilterRegex = value; + else + { + //attempt at being backwards compatible with old way of filtering + var re = string.Join("|", value.Split(new[] { ','}, RemoveEmptyEntries).Select(s => s.Trim())); + PartitionFilterRegex = re; + } + } + } + + /// + /// A global test filter that can be used to only run certain cluster's tests. + /// Accepts a comma separated list of filters + /// + [Obsolete("Use ParitionFilterRegex instead", false)] + public string TestFilter + { + get => TestFilterRegex; + set + { + if (string.IsNullOrWhiteSpace(value)) TestFilterRegex = value; + else + { + //attempt at being backwards compatible with old way of filtering + var re = string.Join("|", value.Split(new[] { ','}, RemoveEmptyEntries).Select(s => s.Trim())); + TestFilterRegex = re; + } + } + } /// /// Informs the runner what version of Elasticsearch is under test. Required for @@ -58,8 +91,10 @@ public override void SetOptions(ITestFrameworkDiscoveryOptions discoveryOptions) IntegrationTestsMayUseAlreadyRunningNode ); discoveryOptions.SetValue(nameof(RunUnitTests), RunUnitTests); +#pragma warning disable CS0618 // Type or member is obsolete discoveryOptions.SetValue(nameof(TestFilter), TestFilter); discoveryOptions.SetValue(nameof(ClusterFilter), ClusterFilter); +#pragma warning restore CS0618 // Type or member is obsolete } public override void SetOptions(ITestFrameworkExecutionOptions executionOptions) @@ -73,8 +108,10 @@ public override void SetOptions(ITestFrameworkExecutionOptions executionOptions) IntegrationTestsMayUseAlreadyRunningNode ); executionOptions.SetValue(nameof(RunUnitTests), RunUnitTests); +#pragma warning disable CS0618 // Type or member is obsolete executionOptions.SetValue(nameof(TestFilter), TestFilter); executionOptions.SetValue(nameof(ClusterFilter), ClusterFilter); +#pragma warning restore CS0618 // Type or member is obsolete } } diff --git a/src/Elastic.Elasticsearch.Xunit/Sdk/ElasticTestFramework.cs b/src/Elastic.Elasticsearch.Xunit/Sdk/ElasticTestFramework.cs index c93c9e7..0b5b3a7 100644 --- a/src/Elastic.Elasticsearch.Xunit/Sdk/ElasticTestFramework.cs +++ b/src/Elastic.Elasticsearch.Xunit/Sdk/ElasticTestFramework.cs @@ -3,14 +3,25 @@ // See the LICENSE file in the project root for more information using System.Reflection; +using Elastic.Elasticsearch.Xunit.XunitPlumbing; using Xunit.Abstractions; using Xunit.Sdk; -using Elastic.Xunit; +using Nullean.Xunit.Partitions; +using Nullean.Xunit.Partitions.Sdk; -namespace Elastic.Elasticsearch.Xunit.Sdk +namespace Elastic.Elasticsearch.Xunit.Sdk; + +// ReSharper disable once UnusedType.Global +public class ElasticTestFramework : PartitionTestFramework +{ + public ElasticTestFramework(IMessageSink messageSink) : base(messageSink) { } +} + +public class TestFrameworkDiscovererFactory : ITestFrameworkDiscovererFactory { - public class ElasticTestFramework : PartitioningTestFramework - { - public ElasticTestFramework(IMessageSink messageSink) : base(messageSink) { } - } + public XunitTestFrameworkDiscoverer Create( + IAssemblyInfo assemblyInfo, ISourceInformationProvider sourceProvider, IMessageSink diagnosticMessageSink + ) + where TOptions : PartitionOptions, new() => + new PartitionTestFrameworkDiscoverer(assemblyInfo, sourceProvider, diagnosticMessageSink, typeof(IClusterFixture<>)); } diff --git a/src/Elastic.Elasticsearch.Xunit/Sdk/TestAssemblyRunner.cs b/src/Elastic.Elasticsearch.Xunit/Sdk/TestAssemblyRunner.cs index 89f626d..d8268bb 100644 --- a/src/Elastic.Elasticsearch.Xunit/Sdk/TestAssemblyRunner.cs +++ b/src/Elastic.Elasticsearch.Xunit/Sdk/TestAssemblyRunner.cs @@ -13,7 +13,7 @@ using Elastic.Elasticsearch.Ephemeral; using Elastic.Elasticsearch.Ephemeral.Tasks.ValidationTasks; using Elastic.Elasticsearch.Xunit.XunitPlumbing; -using Elastic.Xunit; +using Nullean.Xunit.Partitions.Sdk; using Xunit.Abstractions; using Xunit.Sdk; @@ -28,8 +28,7 @@ public XunitTestAssemblyRunner Create(ITestAssembly testAssembly, IEnumerable> + internal class TestAssemblyRunner : PartitionTestAssemblyRunner> { public TestAssemblyRunner(ITestAssembly testAssembly, IEnumerable testCases, diff --git a/src/Elastic.Elasticsearch.Xunit/XunitPlumbing/SkippingTestCase.cs b/src/Elastic.Elasticsearch.Xunit/XunitPlumbing/SkippingTestCase.cs index 2cd54a6..9cb7d21 100644 --- a/src/Elastic.Elasticsearch.Xunit/XunitPlumbing/SkippingTestCase.cs +++ b/src/Elastic.Elasticsearch.Xunit/XunitPlumbing/SkippingTestCase.cs @@ -12,9 +12,8 @@ namespace Elastic.Elasticsearch.Xunit.XunitPlumbing public class SkippingTestCase : TestMethodTestCase, IXunitTestCase { /// Used for de-serialization. - public SkippingTestCase() - { - } + // ReSharper disable once UnusedMember.Global + public SkippingTestCase() { } /// /// Initializes a new instance of the class. diff --git a/src/Elastic.Xunit/Elastic.Xunit.csproj b/src/Elastic.Xunit/Elastic.Xunit.csproj deleted file mode 100644 index 4db8923..0000000 --- a/src/Elastic.Xunit/Elastic.Xunit.csproj +++ /dev/null @@ -1,14 +0,0 @@ - - - - netstandard2.0;netstandard2.1;net462 - enable - enable - false - - - - - - - diff --git a/src/Elastic.Xunit/ForEachAsyncExtensions.cs b/src/Elastic.Xunit/ForEachAsyncExtensions.cs deleted file mode 100644 index 6224bc7..0000000 --- a/src/Elastic.Xunit/ForEachAsyncExtensions.cs +++ /dev/null @@ -1,24 +0,0 @@ -// Licensed to Elasticsearch B.V under one or more agreements. -// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. -// See the LICENSE file in the project root for more information - -using System; -using System.Collections.Concurrent; -using System.Collections.Generic; -using System.Linq; -using System.Threading.Tasks; - -namespace Elastic.Xunit; - -internal static class ForEachAsyncExtensions -{ - internal static Task ForEachAsync(this IEnumerable source, int dop, Func body) => - Task.WhenAll( - from partition in Partitioner.Create(source).GetPartitions(dop) - select Task.Run(async delegate - { - using (partition) - while (partition.MoveNext()) - await body(partition.Current).ConfigureAwait(false); - })); -} diff --git a/src/Elastic.Xunit/PartitioningConfigurationAttribute.cs b/src/Elastic.Xunit/PartitioningConfigurationAttribute.cs deleted file mode 100644 index 8a88da4..0000000 --- a/src/Elastic.Xunit/PartitioningConfigurationAttribute.cs +++ /dev/null @@ -1,45 +0,0 @@ -// Licensed to Elasticsearch B.V under one or more agreements. -// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. -// See the LICENSE file in the project root for more information - -using System; -using System.Linq; -using System.Reflection; - -namespace Elastic.Xunit; - -/// -/// An assembly attribute that specifies the -/// for Xunit tests within the assembly. -/// -[AttributeUsage(AttributeTargets.Assembly)] -public class PartitioningConfigurationAttribute : Attribute -{ - private readonly Type _type; - - /// - /// Creates a new instance of - /// - /// - /// A type deriving from that specifies the run options - /// - public PartitioningConfigurationAttribute(Type type) => _type = type; - - private TOptions GetOptions() where TOptions : PartitioningRunOptions, new() - { - var options = Activator.CreateInstance(_type) as TOptions; - return options ?? new TOptions(); - } - - public static TOptions GetOptions(Assembly assembly) where TOptions : PartitioningRunOptions, new() - { - var options = assembly - .GetCustomAttributes() - .OfType() - .FirstOrDefault() - ?.GetOptions() - ?? new TOptions(); - - return options; - } -} diff --git a/src/Elastic.Xunit/PartitioningRunOptions.cs b/src/Elastic.Xunit/PartitioningRunOptions.cs deleted file mode 100644 index 022257d..0000000 --- a/src/Elastic.Xunit/PartitioningRunOptions.cs +++ /dev/null @@ -1,57 +0,0 @@ -// Licensed to Elasticsearch B.V under one or more agreements. -// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. -// See the LICENSE file in the project root for more information - -using System; -using System.Collections.Concurrent; -using System.Collections.Generic; -using System.Diagnostics; -using Xunit.Abstractions; - -namespace Elastic.Xunit; - -/// -/// The Xunit test runner options -/// -public class PartitioningRunOptions -{ - /// - /// A global test filter that can be used to only run certain tests. - /// Accepts a comma separated list of filters - /// - public string? TestFilter { get; set; } - - public string? GroupFilter { get; set; } - - /// - /// Called when the tests have finished running successfully - /// - /// Per cluster timings of the total test time, including starting Elasticsearch - /// All collection of failed cluster, failed tests tuples - public virtual void OnTestsFinished( - Dictionary partitionTimings, - ConcurrentBag> failedPartitionTests) - { - } - - /// - /// Called before tests run. An ideal place to perform actions such as writing information to - /// . - /// - public virtual void OnBeforeTestsRun() - { - } - - public virtual void SetOptions(ITestFrameworkDiscoveryOptions discoveryOptions) - { - discoveryOptions.SetValue(nameof(GroupFilter), GroupFilter); - discoveryOptions.SetValue(nameof(TestFilter), TestFilter); - } - - public virtual void SetOptions(ITestFrameworkExecutionOptions executionOptions) - { - executionOptions.SetValue(nameof(GroupFilter), GroupFilter); - executionOptions.SetValue(nameof(GroupFilter), TestFilter); - - } -} diff --git a/src/Elastic.Xunit/PartitioningTestAssemblyRunner.cs b/src/Elastic.Xunit/PartitioningTestAssemblyRunner.cs deleted file mode 100644 index 1c8d9c0..0000000 --- a/src/Elastic.Xunit/PartitioningTestAssemblyRunner.cs +++ /dev/null @@ -1,310 +0,0 @@ -// Licensed to Elasticsearch B.V under one or more agreements. -// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. -// See the LICENSE file in the project root for more information - -using System; -using System.Collections.Concurrent; -using System.Collections.Generic; -using System.Diagnostics; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; -using Xunit; -using Xunit.Abstractions; -using Xunit.Sdk; - -namespace Elastic.Xunit -{ - public interface ITestAssemblyRunnerFactory - { - public XunitTestAssemblyRunner Create( - ITestAssembly testAssembly, - IEnumerable testCases, - IMessageSink diagnosticMessageSink, - IMessageSink executionMessageSink, - ITestFrameworkExecutionOptions executionOptions - ); - } - - public class PartitioningTestRunnerFactory : ITestAssemblyRunnerFactory - { - public XunitTestAssemblyRunner Create(ITestAssembly testAssembly, IEnumerable testCases, - IMessageSink diagnosticMessageSink, - IMessageSink executionMessageSink, ITestFrameworkExecutionOptions executionOptions) => - new PartitioningTestAssemblyRunner(testAssembly, testCases, diagnosticMessageSink, executionMessageSink, - executionOptions); - } - - public interface ITestPartition : IAsyncLifetime - { - public int? MaxConcurrency { get; } - } - - // ReSharper disable once UnusedTypeParameter - public interface IPartitionFixture where TPartition : ITestPartition { } - - public class PartitioningTestAssemblyRunner : PartitioningTestAssemblyRunner - { - public PartitioningTestAssemblyRunner( - ITestAssembly testAssembly, - IEnumerable testCases, - IMessageSink diagnosticMessageSink, - IMessageSink executionMessageSink, - ITestFrameworkExecutionOptions executionOptions) - : base(testAssembly, testCases, diagnosticMessageSink, executionMessageSink, executionOptions, - typeof(IPartitionFixture<>)) - { - } - - protected override async Task UseStateAndRun(ITestPartition partition, Func runGroup) - { - await using (partition) - { - await partition.InitializeAsync().ConfigureAwait(false); - await runGroup(partition.MaxConcurrency).ConfigureAwait(false); - } - } - } - - public record TestType - { - public Type? Type { get; set; } - } - - public class TestPartitionGrouping - { - public Type? Type { get; set; } - public ITestCollection Collection { get; set; } = null!; - public List TestCases { get; set; } = null!; - } - - public abstract class PartitioningTestAssemblyRunner : XunitTestAssemblyRunner - where TState : class - { - private readonly Type _fixtureType; - private readonly Dictionary _partitionFixtureInstances = new(); - - protected Dictionary> Partionings { get; } - - protected PartitioningTestAssemblyRunner(ITestAssembly testAssembly, - IEnumerable testCases, - IMessageSink diagnosticMessageSink, - IMessageSink executionMessageSink, - ITestFrameworkExecutionOptions executionOptions, Type fixtureType) - : base(testAssembly, testCases, diagnosticMessageSink, executionMessageSink, executionOptions) - { - _fixtureType = fixtureType; - GroupFilter = executionOptions.GetValue(nameof(PartitioningRunOptions.GroupFilter)); - TestFilter = executionOptions.GetValue(nameof(PartitioningRunOptions.TestFilter)); - - var testCollections = OrderTestCollections(); - - var cases = - from testCollection in testCollections - from classes in testCollection.Item2 - .Select(collection => collection.TestMethod.TestClass.Class) - .Distinct() - let partition = GetPartitionFixtureType(classes) - let testcase = new TestPartitionGrouping - { - Collection = testCollection.Item1, TestCases = testCollection.Item2, Type = partition - } - select testcase; - - Partionings = cases - .GroupBy(c => c.Type) - .OrderBy(g => g.Count()) - .ToDictionary(k => new TestType { Type = k.Key }, v => v.Select(g => g)); - - // types need to be instantiated ahead of time in order for xunit constructor injection checks. - foreach (var partitioning in Partionings) - { - var partitionType = partitioning.Key.Type; - if (partitionType == null) - continue; - - var state = CreatePartitionStateInstance(partitionType); - if (state != null) - _partitionFixtureInstances[partitionType] = state; - } - } - - //threading guess - private static int DefaultConcurrency => Environment.ProcessorCount * 4; - - private ConcurrentBag Summaries { get; } = new(); - public ConcurrentBag> FailedCollections { get; } = new(); - public Dictionary ClusterTotals { get; } = new(); - - private string GroupFilter { get; } - private string TestFilter { get; } - - protected abstract Task UseStateAndRun(TState state, Func runGroup); - - protected override Task RunTestCollectionAsync( - IMessageBus b, - ITestCollection c, - IEnumerable t, CancellationTokenSource s) - { - var aggregator = new ExceptionAggregator(Aggregator); - var fixtureObjects = new Dictionary(); - foreach (var kv in _partitionFixtureInstances) - fixtureObjects.Add(kv.Key, kv.Value); - var runner = new TestCollectionRunner(fixtureObjects, c, t, DiagnosticMessageSink, b, TestCaseOrderer, aggregator, s); - return runner.RunAsync(); - } - - protected async Task RunAllWithoutPartitionFixture(IMessageBus bus, CancellationTokenSource ctx) => - await RunWithoutPartitionFixture(Partionings.SelectMany(g => g.Value), bus, ctx).ConfigureAwait(false); - - protected async Task RunWithoutPartitionFixture( - IEnumerable partitionTests, - IMessageBus messageBus, CancellationTokenSource ctx) - { - var testFilters = CreateTestFilters(TestFilter); - - await RunPartitionGroupConcurrently( - partitionTests, DefaultConcurrency, messageBus, - testFilters, ctx) - .ConfigureAwait(false); - - return new RunSummary - { - Total = Summaries.Sum(s => s.Total), - Failed = Summaries.Sum(s => s.Failed), - Skipped = Summaries.Sum(s => s.Skipped) - }; - } - - protected override async Task RunTestCollectionsAsync(IMessageBus bus, CancellationTokenSource ctx) => - await RunAllTests(bus, ctx).ConfigureAwait(false); - - protected async Task RunAllTests(IMessageBus messageBus, CancellationTokenSource ctx) - { - var testFilters = CreateTestFilters(TestFilter); - foreach (var partitioning in Partionings) - { - var partitionType = partitioning.Key.Type; - if (partitionType == null) - { - var summary =await RunWithoutPartitionFixture(partitioning.Value, messageBus, ctx).ConfigureAwait(false); - Summaries.Add(summary); - continue; - } - - var state = CreatePartitionStateInstance(partitionType); - if (state == null) - { - var testClass = partitioning.Value.Select(g => g.Collection.DisplayName).FirstOrDefault(); - throw new Exception($"{typeof(TState)} did not yield partition state for e.g: {testClass}"); - } - - var type = state.GetType(); - var partitionName = type.Name; - if (!MatchesGroupFilter(partitionName)) continue; - - var skipReasons = partitioning.Value.SelectMany(g => g.TestCases.Select(t => t.SkipReason)).ToList(); - var allSkipped = skipReasons.All(r => !string.IsNullOrWhiteSpace(r)); - if (allSkipped) - { - Summaries.Add(new RunSummary { Total = skipReasons.Count, Skipped = skipReasons.Count }); - continue; - } - - ClusterTotals.Add(partitionName, Stopwatch.StartNew()); - - await UseStateAndRun(state, async (concurrency) => - { - await RunPartitionGroupConcurrently(partitioning.Value, concurrency, messageBus, testFilters, ctx) - .ConfigureAwait(false); - }).ConfigureAwait(false); - - ClusterTotals[partitionName].Stop(); - } - - return new RunSummary - { - Total = Summaries.Sum(s => s.Total), - Failed = Summaries.Sum(s => s.Failed), - Skipped = Summaries.Sum(s => s.Skipped) - }; - } - - private async Task RunPartitionGroupConcurrently( - IEnumerable source, - int? concurrency, - IMessageBus bus, string[] testFilters, CancellationTokenSource ctx) => - await source - .ForEachAsync(Math.Max(concurrency ?? 0, DefaultConcurrency), - async g => await ExecutePartitionGrouping(bus, ctx, g, testFilters).ConfigureAwait(false) - ) - .ConfigureAwait(false); - - private async Task ExecutePartitionGrouping( - IMessageBus messageBus, - CancellationTokenSource ctx, - TestPartitionGrouping g, - string[] testFilters - ) - { - var test = g.Collection.DisplayName.Replace("Test collection for", string.Empty).Trim(); - if (!MatchesATestFilter(test, testFilters)) return; - try - { - var summary = await RunTestCollectionAsync(messageBus, g.Collection, g.TestCases, ctx) - .ConfigureAwait(false); - var type = g.Type; - var partitionName = type?.Name ?? "UNKNOWN"; - if (summary.Failed > 0) - FailedCollections.Add(Tuple.Create(partitionName, test)); - Summaries.Add(summary); - } - catch (TaskCanceledException) - { - // TODO: What should happen here? - } - } - - private static string[] CreateTestFilters(string? testFilters) => - testFilters?.Split(',').Select(s => s.Trim()).Where(s => !string.IsNullOrWhiteSpace(s)).ToArray() - ?? new string[] { }; - - private static bool MatchesATestFilter(string test, IReadOnlyCollection testFilters) - { - if (testFilters.Count == 0 || string.IsNullOrWhiteSpace(test)) return true; - return testFilters - .Any(filter => test.IndexOf(filter, StringComparison.OrdinalIgnoreCase) >= 0); - } - - private bool MatchesGroupFilter(string cluster) - { - if (string.IsNullOrWhiteSpace(cluster) || string.IsNullOrWhiteSpace(GroupFilter)) return true; - return GroupFilter - .Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries) - .Select(c => c.Trim()) - .Any(c => cluster.IndexOf(c, StringComparison.OrdinalIgnoreCase) >= 0); - } - - private TState? CreatePartitionStateInstance(Type partitionType) - { - if (_partitionFixtureInstances.TryGetValue(partitionType, out var partition)) return partition; - Aggregator.Run(() => - { - var o = Activator.CreateInstance(partitionType); - partition = o as TState; - }); - _partitionFixtureInstances.Add(partitionType, partition); - return partition; - } - - private Type? GetPartitionFixtureType(ITypeInfo testClass) => - GetPartitionFixtureType(testClass, _fixtureType); - - protected static Type? GetPartitionFixtureType(ITypeInfo testClass, Type openGenericState) => - testClass.ToRuntimeType().GetInterfaces() - .Where(i => i.IsGenericType) - .Where(t => t.GetGenericTypeDefinition() == openGenericState) - .SelectMany(t => t.GetGenericArguments(), (_, a) => a) - .FirstOrDefault(); - } -} diff --git a/src/Elastic.Xunit/PartitioningTestFramework.cs b/src/Elastic.Xunit/PartitioningTestFramework.cs deleted file mode 100644 index 5e9c762..0000000 --- a/src/Elastic.Xunit/PartitioningTestFramework.cs +++ /dev/null @@ -1,36 +0,0 @@ -// Licensed to Elasticsearch B.V under one or more agreements. -// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. -// See the LICENSE file in the project root for more information - -using System.Reflection; -using Xunit.Abstractions; -using Xunit.Sdk; - -namespace Elastic.Xunit; - -// ReSharper disable once UnusedType.Global -/// -/// -/// -public class PartitioningTestFramework : PartitioningTestFramework -{ - public PartitioningTestFramework(IMessageSink messageSink) : base(messageSink) { } -} - -public abstract class PartitioningTestFramework : XunitTestFramework - where TOptions : PartitioningRunOptions, new() - where TRunnerFactory : ITestAssemblyRunnerFactory, new() -{ - protected PartitioningTestFramework(IMessageSink messageSink) : base(messageSink) { } - - protected override ITestFrameworkDiscoverer CreateDiscoverer(IAssemblyInfo assemblyInfo) => - new PartitioningTestFrameworkDiscoverer(assemblyInfo, SourceInformationProvider, DiagnosticMessageSink); - - protected override ITestFrameworkExecutor CreateExecutor(AssemblyName assemblyName) - { - var assembly = Assembly.Load(assemblyName); - var options = PartitioningConfigurationAttribute.GetOptions(assembly); - - return new PartitioningTestFrameworkExecutor(options, assemblyName, SourceInformationProvider, DiagnosticMessageSink); - } -} diff --git a/src/Elastic.Xunit/PartitioningTestFrameworkDiscoverer.cs b/src/Elastic.Xunit/PartitioningTestFrameworkDiscoverer.cs deleted file mode 100644 index 9c7b92a..0000000 --- a/src/Elastic.Xunit/PartitioningTestFrameworkDiscoverer.cs +++ /dev/null @@ -1,33 +0,0 @@ -// Licensed to Elasticsearch B.V under one or more agreements. -// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. -// See the LICENSE file in the project root for more information - -using System.Reflection; -using Xunit.Abstractions; -using Xunit.Sdk; - -namespace Elastic.Xunit; - -internal class PartitioningTestFrameworkDiscoverer : XunitTestFrameworkDiscoverer - where TOptions : PartitioningRunOptions, new() -{ - public PartitioningTestFrameworkDiscoverer( - IAssemblyInfo assemblyInfo, - ISourceInformationProvider sourceProvider, - IMessageSink diagnosticMessageSink, - IXunitTestCollectionFactory? collectionFactory = null) : base( - assemblyInfo, sourceProvider, diagnosticMessageSink, collectionFactory) - { - var a = Assembly.Load(new AssemblyName(assemblyInfo.Name)); - Options = PartitioningConfigurationAttribute.GetOptions(a); - } - - private TOptions Options { get; } - - protected override bool FindTestsForType(ITestClass testClass, bool includeSourceInformation, - IMessageBus messageBus, ITestFrameworkDiscoveryOptions discoveryOptions) - { - Options.SetOptions(discoveryOptions); - return base.FindTestsForType(testClass, includeSourceInformation, messageBus, discoveryOptions); - } -} diff --git a/src/Elastic.Xunit/PartitioningTestFrameworkExecutor.cs b/src/Elastic.Xunit/PartitioningTestFrameworkExecutor.cs deleted file mode 100644 index 10ec4ea..0000000 --- a/src/Elastic.Xunit/PartitioningTestFrameworkExecutor.cs +++ /dev/null @@ -1,76 +0,0 @@ -// Licensed to Elasticsearch B.V under one or more agreements. -// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. -// See the LICENSE file in the project root for more information - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Reflection; -using Xunit.Abstractions; -using Xunit.Sdk; - -namespace Elastic.Xunit -{ - internal class PartitioningTestFrameworkExecutor : XunitTestFrameworkExecutor - where TOptions : PartitioningRunOptions, new() - where TRunnerFactory : ITestAssemblyRunnerFactory, new() - { - public PartitioningTestFrameworkExecutor( - TOptions options, - AssemblyName a, - ISourceInformationProvider sip, - IMessageSink d - ) - : base(a, sip, d) - { - Options = options; - RunnerFactory = new TRunnerFactory(); - } - - private ITestAssemblyRunnerFactory RunnerFactory { get; } - - private TOptions Options { get; } - - public override void RunAll( - IMessageSink executionMessageSink, - ITestFrameworkDiscoveryOptions discoveryOptions, - ITestFrameworkExecutionOptions executionOptions - ) - { - Options.SetOptions(discoveryOptions); - Options.SetOptions(executionOptions); - - base.RunAll(executionMessageSink, discoveryOptions, executionOptions); - } - - public override void RunTests( - IEnumerable testCases, - IMessageSink executionMessageSink, - ITestFrameworkExecutionOptions executionOptions) - { - Options.SetOptions(executionOptions); - base.RunTests(testCases, executionMessageSink, executionOptions); - } - - protected override async void RunTestCases( - IEnumerable testCases, - IMessageSink sink, - ITestFrameworkExecutionOptions options) - { - Options.SetOptions(options); - try - { - using var runner = RunnerFactory.Create(TestAssembly, testCases, DiagnosticMessageSink, sink, options); - Options.OnBeforeTestsRun(); - await runner.RunAsync().ConfigureAwait(false); - if (runner is PartitioningTestAssemblyRunner a) - Options.OnTestsFinished(a.ClusterTotals, a.FailedCollections); - } - catch (Exception e) - { - sink.OnMessage(new TestAssemblyCleanupFailure(Enumerable.Empty(), TestAssembly, e)); - throw; - } - } - } -} diff --git a/src/Elastic.Xunit/TestCollectionRunner.cs b/src/Elastic.Xunit/TestCollectionRunner.cs deleted file mode 100644 index 5530d9d..0000000 --- a/src/Elastic.Xunit/TestCollectionRunner.cs +++ /dev/null @@ -1,47 +0,0 @@ -// Licensed to Elasticsearch B.V under one or more agreements. -// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. -// See the LICENSE file in the project root for more information - -using System; -using System.Collections.Generic; -using System.Threading; -using System.Threading.Tasks; -using Xunit.Abstractions; -using Xunit.Sdk; - -namespace Elastic.Xunit; - -internal class TestCollectionRunner : XunitTestCollectionRunner -{ - private readonly Dictionary _assemblyFixtureMappings; - private readonly IMessageSink _diagnosticMessageSink; - - public TestCollectionRunner(Dictionary assemblyFixtureMappings, - ITestCollection testCollection, - IEnumerable testCases, - IMessageSink diagnosticMessageSink, - IMessageBus messageBus, - ITestCaseOrderer testCaseOrderer, - ExceptionAggregator aggregator, - CancellationTokenSource cancellationTokenSource) - : base(testCollection, testCases, diagnosticMessageSink, messageBus, testCaseOrderer, aggregator, - cancellationTokenSource) - { - _assemblyFixtureMappings = assemblyFixtureMappings; - _diagnosticMessageSink = diagnosticMessageSink; - } - - protected override Task RunTestClassAsync(ITestClass testClass, IReflectionTypeInfo @class, - IEnumerable testCases) - { - // this ensures xunit can constructor inject our partition types - var combinedFixtures = new Dictionary(_assemblyFixtureMappings); - foreach (var kvp in CollectionFixtureMappings) - combinedFixtures[kvp.Key] = kvp.Value; - - // We've done everything we need, so hand back off to default Xunit implementation for class runner - return new XunitTestClassRunner(testClass, @class, testCases, _diagnosticMessageSink, MessageBus, - TestCaseOrderer, new ExceptionAggregator(Aggregator), CancellationTokenSource, combinedFixtures) - .RunAsync(); - } -} From 0c5661f81d2b25389dca57f3449c455fe65ea6c1 Mon Sep 17 00:00:00 2001 From: Martijn Laarman Date: Fri, 9 Feb 2024 15:36:07 +0100 Subject: [PATCH 4/5] Move to Nullean.Xunit.Partitions --- .../Elastic.Elasticsearch.Xunit.csproj | 3 ++- src/Elastic.Elasticsearch.Xunit/ElasticXunitRunOptions.cs | 3 +++ src/Elastic.Elasticsearch.Xunit/Sdk/ElasticTestFramework.cs | 4 ++-- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/Elastic.Elasticsearch.Xunit/Elastic.Elasticsearch.Xunit.csproj b/src/Elastic.Elasticsearch.Xunit/Elastic.Elasticsearch.Xunit.csproj index ccc07f1..3d7b83e 100644 --- a/src/Elastic.Elasticsearch.Xunit/Elastic.Elasticsearch.Xunit.csproj +++ b/src/Elastic.Elasticsearch.Xunit/Elastic.Elasticsearch.Xunit.csproj @@ -2,11 +2,12 @@ netstandard2.0;netstandard2.1;net462 True + False Provides an Xunit test framework allowing you to run integration tests against local ephemeral Elasticsearch clusters elastic,elasticsearch,xunit,cluster,integration,test,ephemeral - + diff --git a/src/Elastic.Elasticsearch.Xunit/ElasticXunitRunOptions.cs b/src/Elastic.Elasticsearch.Xunit/ElasticXunitRunOptions.cs index 395aae4..5215d3b 100644 --- a/src/Elastic.Elasticsearch.Xunit/ElasticXunitRunOptions.cs +++ b/src/Elastic.Elasticsearch.Xunit/ElasticXunitRunOptions.cs @@ -4,6 +4,7 @@ using System; using System.Linq; +using System.Runtime.Serialization; using Elastic.Elasticsearch.Xunit.XunitPlumbing; using Elastic.Stack.ArtifactsApi; using Nullean.Xunit.Partitions; @@ -40,6 +41,7 @@ public class ElasticXunitRunOptions : PartitionOptions /// Accepts a comma separated list of filters /// [Obsolete("Use PartitionFilterRegex instead", false)] + [IgnoreDataMember] public string ClusterFilter { get => PartitionFilterRegex; @@ -60,6 +62,7 @@ public string ClusterFilter /// Accepts a comma separated list of filters /// [Obsolete("Use ParitionFilterRegex instead", false)] + [IgnoreDataMember] public string TestFilter { get => TestFilterRegex; diff --git a/src/Elastic.Elasticsearch.Xunit/Sdk/ElasticTestFramework.cs b/src/Elastic.Elasticsearch.Xunit/Sdk/ElasticTestFramework.cs index 0b5b3a7..baf0588 100644 --- a/src/Elastic.Elasticsearch.Xunit/Sdk/ElasticTestFramework.cs +++ b/src/Elastic.Elasticsearch.Xunit/Sdk/ElasticTestFramework.cs @@ -12,12 +12,12 @@ namespace Elastic.Elasticsearch.Xunit.Sdk; // ReSharper disable once UnusedType.Global -public class ElasticTestFramework : PartitionTestFramework +public class ElasticTestFramework : PartitionTestFramework { public ElasticTestFramework(IMessageSink messageSink) : base(messageSink) { } } -public class TestFrameworkDiscovererFactory : ITestFrameworkDiscovererFactory +public class ElasticTestFrameworkDiscovererFactory : ITestFrameworkDiscovererFactory { public XunitTestFrameworkDiscoverer Create( IAssemblyInfo assemblyInfo, ISourceInformationProvider sourceProvider, IMessageSink diagnosticMessageSink From 5adf9335f80814ac8d5cc3edb40a91b607dfb714 Mon Sep 17 00:00:00 2001 From: Martijn Laarman Date: Fri, 23 Feb 2024 16:22:22 +0100 Subject: [PATCH 5/5] update to latest nullean.xunit.partitions --- .../Elastic.Elasticsearch.Xunit.csproj | 2 +- .../Sdk/TestAssemblyRunner.cs | 24 +++++++++++++++---- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/src/Elastic.Elasticsearch.Xunit/Elastic.Elasticsearch.Xunit.csproj b/src/Elastic.Elasticsearch.Xunit/Elastic.Elasticsearch.Xunit.csproj index 3d7b83e..6015f45 100644 --- a/src/Elastic.Elasticsearch.Xunit/Elastic.Elasticsearch.Xunit.csproj +++ b/src/Elastic.Elasticsearch.Xunit/Elastic.Elasticsearch.Xunit.csproj @@ -7,7 +7,7 @@ elastic,elasticsearch,xunit,cluster,integration,test,ephemeral - + diff --git a/src/Elastic.Elasticsearch.Xunit/Sdk/TestAssemblyRunner.cs b/src/Elastic.Elasticsearch.Xunit/Sdk/TestAssemblyRunner.cs index d8268bb..798fe16 100644 --- a/src/Elastic.Elasticsearch.Xunit/Sdk/TestAssemblyRunner.cs +++ b/src/Elastic.Elasticsearch.Xunit/Sdk/TestAssemblyRunner.cs @@ -59,17 +59,33 @@ protected override async Task RunTestCollectionsAsync(IMessageBus bu .ConfigureAwait(false); } - protected override async Task UseStateAndRun(IEphemeralCluster cluster, Func runGroup) + protected override async Task UseStateAndRun( + IEphemeralCluster cluster, + Func runGroup, + Func failAll + ) { using (cluster) { ElasticXunitRunner.CurrentCluster = cluster; var clusterConfiguration = cluster.ClusterConfiguration; var timeout = clusterConfiguration?.Timeout ?? TimeSpan.FromMinutes(2); - if (!IntegrationTestsMayUseAlreadyRunningNode || !ValidateRunningVersion(cluster)) - cluster.Start(timeout); - await runGroup(clusterConfiguration?.MaxConcurrency).ConfigureAwait(false); + var started = false; + try + { + if (!IntegrationTestsMayUseAlreadyRunningNode || !ValidateRunningVersion(cluster)) + cluster.Start(timeout); + + started = true; + } + catch (Exception e) + { + await failAll(e, $"Further logs might be available at: {cluster.ClusterConfiguration?.FileSystem?.LogsPath}") + .ConfigureAwait(false); + } + if (started) + await runGroup(clusterConfiguration?.MaxConcurrency).ConfigureAwait(false); } }