From 0ebf19db40dea6461ec478806f45c019e34c061d Mon Sep 17 00:00:00 2001 From: Mike Curn Date: Mon, 27 Mar 2023 11:45:54 -0700 Subject: [PATCH] Testbuilder v2 (#114) * Lots of (admittedly poorly managed) changes to rev to v2. * Biggest change: Now there's a TestBuilder.Build() method that will validate as many dependencies as possible. --------- Co-authored-by: Michael Curn Co-authored-by: Kevin B --- .../ExampleTests/ExampleTests.csproj | 27 +- .../ExampleTests/Harness/BasePage.cs | 5 +- .../ExampleTests/IntelliTectTests.cs | 6 +- .../ExampleTests/ExampleTests/NewLogger.cs | 2 +- .../ExampleTests/WebDriverFactory.cs | 7 +- .../ErrorMessages.cs | 11 + .../ExampleDataThing.cs | 12 - .../ExampleDataThingFactory.cs | 19 - .../ExampleLogger.cs | 42 ++ ...iTect.TestTools.TestFramework.Tests.csproj | 22 +- .../TestBase.cs | 22 + .../TestBuilderTests.cs | 544 ------------------ .../ErrorConditions/ExecuteMethodErrors.cs | 54 ++ .../ErrorConditions/ExecuteOverrideTests.cs | 126 ++++ .../MultipleDependencyErrors.cs | 112 ++++ .../ErrorConditions/SingleDependencyErrors.cs | 125 ++++ .../PositiveConditions/LoggerTests.cs | 188 ++++++ .../PositiveConditions/ResolverTests.cs | 80 +++ .../TestCasePropertyTests.cs | 170 ++++++ .../TestCaseTests/ExecuteArgumentOverrides.cs | 80 +++ .../TestCaseTests/FinallyExecutionTests.cs | 91 +++ .../TestCaseTests/MultipleDependencyTests.cs | 42 ++ .../TestCaseTests/SingleDependencyTests.cs | 146 +++++ .../TestCaseTests/TestFailureTests.cs | 44 ++ .../TestData/Dependencies/ExampleFactory.cs | 44 ++ .../TestData/Dependencies/ExampleInterface.cs | 17 + .../TestData/Dependencies/SimulatorClasses.cs | 111 ++++ .../TestBlocks/MultipleDependencyBlocks.cs | 24 + .../TestBlocks/SingleDependencyBlocks.cs | 73 +++ .../Block.cs | 23 + .../DebugLogger.cs | 68 +++ .../ILogger.cs | 20 - .../ITestBlock.cs | 8 +- .../ITestCaseLogger.cs | 13 + ...IntelliTect.TestTools.TestFramework.csproj | 20 +- .../Log.cs | 38 -- .../TestBlock.cs | 7 + .../TestBuilder.cs | 413 ++++++------- .../TestCase.cs | 432 ++++++++++++++ .../TestCaseException.cs | 4 +- testframework-pipeline.yml | 2 +- 41 files changed, 2392 insertions(+), 902 deletions(-) create mode 100644 IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework.Tests/ErrorMessages.cs delete mode 100644 IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework.Tests/ExampleDataThing.cs delete mode 100644 IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework.Tests/ExampleDataThingFactory.cs create mode 100644 IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework.Tests/ExampleLogger.cs create mode 100644 IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework.Tests/TestBase.cs delete mode 100644 IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework.Tests/TestBuilderTests.cs create mode 100644 IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework.Tests/TestBuilderTests/ErrorConditions/ExecuteMethodErrors.cs create mode 100644 IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework.Tests/TestBuilderTests/ErrorConditions/ExecuteOverrideTests.cs create mode 100644 IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework.Tests/TestBuilderTests/ErrorConditions/MultipleDependencyErrors.cs create mode 100644 IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework.Tests/TestBuilderTests/ErrorConditions/SingleDependencyErrors.cs create mode 100644 IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework.Tests/TestBuilderTests/PositiveConditions/LoggerTests.cs create mode 100644 IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework.Tests/TestBuilderTests/PositiveConditions/ResolverTests.cs create mode 100644 IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework.Tests/TestBuilderTests/PositiveConditions/TestCasePropertyTests.cs create mode 100644 IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework.Tests/TestCaseTests/ExecuteArgumentOverrides.cs create mode 100644 IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework.Tests/TestCaseTests/FinallyExecutionTests.cs create mode 100644 IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework.Tests/TestCaseTests/MultipleDependencyTests.cs create mode 100644 IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework.Tests/TestCaseTests/SingleDependencyTests.cs create mode 100644 IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework.Tests/TestCaseTests/TestFailureTests.cs create mode 100644 IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework.Tests/TestData/Dependencies/ExampleFactory.cs create mode 100644 IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework.Tests/TestData/Dependencies/ExampleInterface.cs create mode 100644 IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework.Tests/TestData/Dependencies/SimulatorClasses.cs create mode 100644 IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework.Tests/TestData/TestBlocks/MultipleDependencyBlocks.cs create mode 100644 IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework.Tests/TestData/TestBlocks/SingleDependencyBlocks.cs create mode 100644 IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework/Block.cs create mode 100644 IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework/DebugLogger.cs delete mode 100644 IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework/ILogger.cs create mode 100644 IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework/ITestCaseLogger.cs delete mode 100644 IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework/Log.cs create mode 100644 IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework/TestBlock.cs create mode 100644 IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework/TestCase.cs diff --git a/IntelliTect.TestTools.TestFramework/ExampleTests/ExampleTests/ExampleTests.csproj b/IntelliTect.TestTools.TestFramework/ExampleTests/ExampleTests/ExampleTests.csproj index 03d0f2aa..ed8c8010 100644 --- a/IntelliTect.TestTools.TestFramework/ExampleTests/ExampleTests/ExampleTests.csproj +++ b/IntelliTect.TestTools.TestFramework/ExampleTests/ExampleTests/ExampleTests.csproj @@ -1,17 +1,28 @@ - + - netcoreapp3.1 - + net6.0 + 11.0 + enable + + CA1303;CS1822;CS1822;CS8620;CS8625;CS8600;CS8602;CS8618;CS8603;CS8604;CA1822;CA1062 + false - - - - - + + all + + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + diff --git a/IntelliTect.TestTools.TestFramework/ExampleTests/ExampleTests/Harness/BasePage.cs b/IntelliTect.TestTools.TestFramework/ExampleTests/ExampleTests/Harness/BasePage.cs index 39924451..7e0f97cc 100644 --- a/IntelliTect.TestTools.TestFramework/ExampleTests/ExampleTests/Harness/BasePage.cs +++ b/IntelliTect.TestTools.TestFramework/ExampleTests/ExampleTests/Harness/BasePage.cs @@ -1,4 +1,5 @@ -using System.Text.Json.Serialization; +using System; +using System.Text.Json.Serialization; using OpenQA.Selenium; namespace ExampleTests.Harness @@ -10,7 +11,7 @@ public BasePage(IWebDriver driver) Driver = driver; } - public string BaseUrl { get; set; } = @"https://intellitect.com/"; + public Uri BaseUrl { get; set; } = new Uri("https://intellitect.com/"); [JsonIgnore] protected IWebDriver Driver { get; set; } } diff --git a/IntelliTect.TestTools.TestFramework/ExampleTests/ExampleTests/IntelliTectTests.cs b/IntelliTect.TestTools.TestFramework/ExampleTests/ExampleTests/IntelliTectTests.cs index 15802a85..d48e71c9 100644 --- a/IntelliTect.TestTools.TestFramework/ExampleTests/ExampleTests/IntelliTectTests.cs +++ b/IntelliTect.TestTools.TestFramework/ExampleTests/ExampleTests/IntelliTectTests.cs @@ -25,7 +25,7 @@ public void Test1() IsBodyAvailable = true }; - TestBuilder builder = new TestBuilder(); + TestBuilder builder = new(); builder .AddLogger() .AddDependencyService(new WebDriverFactory("Chrome").Driver) @@ -38,7 +38,7 @@ public void Test1() [Fact] public void RegisterMembership() { - TestBuilder builder = new TestBuilder(); + TestBuilder builder = new(); builder .AddDependencyService(new WebDriverFactory("Chrome").Driver) .AddDependencyService() @@ -50,7 +50,7 @@ public void RegisterMembership() [Fact] public void LogIn() { - TestBuilder builder = new TestBuilder(); + TestBuilder builder = new(); builder .AddDependencyService(new WebDriverFactory("Chrome").Driver) .AddDependencyService(new AccountFactory().Account) diff --git a/IntelliTect.TestTools.TestFramework/ExampleTests/ExampleTests/NewLogger.cs b/IntelliTect.TestTools.TestFramework/ExampleTests/ExampleTests/NewLogger.cs index c3302941..7cdbfb62 100644 --- a/IntelliTect.TestTools.TestFramework/ExampleTests/ExampleTests/NewLogger.cs +++ b/IntelliTect.TestTools.TestFramework/ExampleTests/ExampleTests/NewLogger.cs @@ -5,7 +5,7 @@ namespace ExampleTests { - class NewLogger : ILogger + public class NewLogger : ILogger { public string TestCaseKey { get; set; } public string CurrentTestBlock { get; set; } diff --git a/IntelliTect.TestTools.TestFramework/ExampleTests/ExampleTests/WebDriverFactory.cs b/IntelliTect.TestTools.TestFramework/ExampleTests/ExampleTests/WebDriverFactory.cs index cbb9ba7d..e47bf7a9 100644 --- a/IntelliTect.TestTools.TestFramework/ExampleTests/ExampleTests/WebDriverFactory.cs +++ b/IntelliTect.TestTools.TestFramework/ExampleTests/ExampleTests/WebDriverFactory.cs @@ -17,14 +17,9 @@ public WebDriverFactory(string browserType) private IWebDriver GetWebDriver(IServiceProvider service) { - if (_BrowserType == "Chrome") - _Driver = new ChromeDriver(Directory.GetCurrentDirectory()); - else - _Driver = new ChromeDriver(Directory.GetCurrentDirectory()); - return _Driver; + return new ChromeDriver(Directory.GetCurrentDirectory()); } - private IWebDriver _Driver; private string _BrowserType; } } diff --git a/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework.Tests/ErrorMessages.cs b/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework.Tests/ErrorMessages.cs new file mode 100644 index 00000000..2b5d50f4 --- /dev/null +++ b/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework.Tests/ErrorMessages.cs @@ -0,0 +1,11 @@ +namespace IntelliTect.TestTools.TestFramework.Tests +{ + public static class ErrorMessages + { + public const string ExecuteError = "there must be one and only one execute method"; + public const string MissingInputError = "unable to satisfy test block input"; + public const string MismatchedExecuteOverrideError = "unable to find corresponding execute parameter"; + public const string TooManyExecuteOverridesError = "too many execute overrides were provided"; + public const string AlreadyAddedError = "multiple execute argument overrides of the same type are not allowed"; + } +} diff --git a/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework.Tests/ExampleDataThing.cs b/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework.Tests/ExampleDataThing.cs deleted file mode 100644 index 2bb22bc1..00000000 --- a/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework.Tests/ExampleDataThing.cs +++ /dev/null @@ -1,12 +0,0 @@ -namespace IntelliTect.TestTools.TestFramework.Tests -{ - public interface IExampleDataInterface - { - string Testing { get; set; } - } - - public class ExampleDataThing : IExampleDataInterface - { - public string Testing { get; set; } = "Testing"; - } -} diff --git a/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework.Tests/ExampleDataThingFactory.cs b/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework.Tests/ExampleDataThingFactory.cs deleted file mode 100644 index fdbcec20..00000000 --- a/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework.Tests/ExampleDataThingFactory.cs +++ /dev/null @@ -1,19 +0,0 @@ -using System; - -namespace IntelliTect.TestTools.TestFramework.Tests -{ - public class ExampleDataThingFactory - { - public ExampleDataThingFactory() - { - ExampleDataThing = GetExampleObject; - } - - public Func ExampleDataThing { get; private set; } - - private ExampleDataThing GetExampleObject(IServiceProvider service) - { - return new ExampleDataThing { Testing = "TestingOverride" }; - } - } -} diff --git a/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework.Tests/ExampleLogger.cs b/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework.Tests/ExampleLogger.cs new file mode 100644 index 00000000..d34309fc --- /dev/null +++ b/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework.Tests/ExampleLogger.cs @@ -0,0 +1,42 @@ +using System; + +namespace IntelliTect.TestTools.TestFramework.Tests +{ + public class ThrowingLogger : ITestCaseLogger + { + public ThrowingLogger(TestCase tc) + { + TestCase = tc; + } + + public string? TestCaseKey { get; set; } + public string? CurrentTestBlock { get; set; } + + public TestCase TestCase { get; } + + public void Debug(string message) + { + throw new NotImplementedException(); + } + + public void Critical(string message) + { + throw new NotImplementedException(); + } + + public void Info(string message) + { + throw new NotImplementedException(); + } + + public void TestBlockInput(object input) + { + throw new NotImplementedException(); + } + + public void TestBlockOutput(object output) + { + throw new NotImplementedException(); + } + } +} diff --git a/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework.Tests/IntelliTect.TestTools.TestFramework.Tests.csproj b/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework.Tests/IntelliTect.TestTools.TestFramework.Tests.csproj index dae5ed45..730a05a6 100644 --- a/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework.Tests/IntelliTect.TestTools.TestFramework.Tests.csproj +++ b/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework.Tests/IntelliTect.TestTools.TestFramework.Tests.csproj @@ -1,16 +1,26 @@  - netcoreapp3.1 - true - 4 - 1701;1702;CA1822;CA1303 + net6.0 + 11.0 + enable + + CA1303; + CA1822; + false - - + + all + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + diff --git a/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework.Tests/TestBase.cs b/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework.Tests/TestBase.cs new file mode 100644 index 00000000..c12bd268 --- /dev/null +++ b/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework.Tests/TestBase.cs @@ -0,0 +1,22 @@ +using System; +using Xunit; + +namespace IntelliTect.TestTools.TestFramework.Tests +{ + public class TestBase + { + protected static void ValidateAggregateException(AggregateException result, int expectedInnerExceptions, params string[] messages) + { + if (result is null) throw new ArgumentNullException(nameof(result)); + Assert.Equal(expectedInnerExceptions, result.InnerExceptions.Count); + + foreach(string s in messages) + { + Assert.Contains(result.InnerExceptions, + m => m.Message.Contains( + s, + StringComparison.InvariantCultureIgnoreCase)); + } + } + } +} diff --git a/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework.Tests/TestBuilderTests.cs b/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework.Tests/TestBuilderTests.cs deleted file mode 100644 index 144a1241..00000000 --- a/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework.Tests/TestBuilderTests.cs +++ /dev/null @@ -1,544 +0,0 @@ -using System; -using Xunit; -using Xunit.Sdk; - -namespace IntelliTect.TestTools.TestFramework.Tests -{ - public class TestBuilderTests - { - [Fact] - public void FetchByObjectInstanceForExecuteArg() - { - TestBuilder builder = new TestBuilder(); - builder - .AddDependencyInstance("Testing") - .AddTestBlock() - .ExecuteTestCase(); - } - - [Fact] - public void FetchByObjectInstanceForTestBlockProperty() - { - TestBuilder builder = new TestBuilder(); - builder - .AddDependencyInstance("Testing") - .AddTestBlock() - .ExecuteTestCase(); - } - - [Fact] - public void FetchByObjectInstanceForTestBlockConstructor() - { - TestBuilder builder = new TestBuilder(); - builder - .AddDependencyInstance("Testing") - .AddTestBlock() - .ExecuteTestCase(); - } - - [Fact] - public void FetchByObjectInstanceForMultipleDependencies() - { - TestBuilder builder = new TestBuilder(); - builder - .AddDependencyInstance("Testing") - .AddDependencyInstance(1234) - .AddTestBlock() - .ExecuteTestCase(); - } - - [Fact] - public void PassExecuteArgumentsViaAddTestBlockParams() - { - TestBuilder builder = new TestBuilder(); - builder - .AddTestBlock("Testing") - .ExecuteTestCase(); - } - - [Fact] - public void MismatchedCountAddTestBlockParamsAndExecuteArgsFails() - { - TestBuilder builder = new TestBuilder(); - builder.AddTestBlock("Testing", "Testing2"); - - Assert.Throws(() => builder.ExecuteTestCase()); - } - - [Fact] - public void MismatchedTypeAddTestBlockParamsAndExecuteArgsFails() - { - TestBuilder builder = new TestBuilder(); - builder.AddTestBlock(1234); - - Assert.Throws(() => builder.ExecuteTestCase()); - } - - // This test probably isn't necessary. This is MS DI out-of-the-box functionality - [Fact] - public void FetchByServiceForConstructor() - { - TestBuilder builder = new TestBuilder(); - builder - .AddDependencyService() - .AddTestBlock() - .ExecuteTestCase(); - } - - [Fact] - public void FetchByServiceForProperty() - { - TestBuilder builder = new TestBuilder(); - builder - .AddDependencyService() - .AddTestBlock() - .ExecuteTestCase(); - } - - [Fact] - public void FetchByServiceForExecuteArg() - { - TestBuilder builder = new TestBuilder(); - builder - .AddDependencyService() - .AddTestBlock() - .ExecuteTestCase(); - } - - [Fact] - public void FetchByImplementationForExecuteArg() - { - TestBuilder builder = new TestBuilder(); - builder - .AddDependencyInstance(new ExampleDataThing()) - .AddTestBlock() - .ExecuteTestCase(); - } - - [Fact] - public void FetchByImplementationAndTypeForExecuteArg() - { - TestBuilder builder = new TestBuilder(); - builder - .AddDependencyService() - .AddTestBlock() - .ExecuteTestCase(); - } - - // This test probably isn't necessary. This is MS DI out-of-the-box functionality - [Fact] - public void FetchByFactoryForConstructor() - { - TestBuilder builder = new TestBuilder(); - builder - .AddDependencyService(new ExampleDataThingFactory().ExampleDataThing) - .AddTestBlock() - .ExecuteTestCase(); - } - - [Fact] - public void FetchByFactoryForProperty() - { - TestBuilder builder = new TestBuilder(); - builder - .AddDependencyService(new ExampleDataThingFactory().ExampleDataThing) - .AddTestBlock() - .ExecuteTestCase(); - } - - [Fact] - public void FetchByFactoryForExecuteArg() - { - TestBuilder builder = new TestBuilder(); - builder - .AddDependencyService(new ExampleDataThingFactory().ExampleDataThing) - .AddTestBlock() - .ExecuteTestCase(); - } - - // This test probably isn't necessary. This is MS DI out-of-the-box functionality - [Fact] - public void AddTwoServicesOfSameTypeToServiceAndFetch() - { - TestBuilder builder = new TestBuilder(); - builder - .AddDependencyService() - .AddDependencyService() - .AddTestBlock() - .ExecuteTestCase(); - } - - // This test probably isn't necessary. This is MS DI out-of-the-box functionality - [Fact] - public void AddTwoInstancesOfSameTypeToServiceAndFetch() - { - TestBuilder builder = new TestBuilder(); - builder - .AddDependencyInstance(new ExampleDataThing { Testing = "Testing2" }) - .AddDependencyInstance(new ExampleDataThing { Testing = "Testing" }) - .AddTestBlock() - .ExecuteTestCase(); - } - - [Fact] - public void TestBlockWithPropertyWithNoSetterDoesNotThrow() - { - TestBuilder builder = new TestBuilder(); - builder - .AddDependencyInstance("Testing") - .AddTestBlock() - .ExecuteTestCase(); - } - - [Fact] - public void TestBlockWithMultipleExecuteMethodsThrows() - { - TestBuilder builder = new TestBuilder(); - builder - .AddDependencyInstance("Testing") - .AddTestBlock(); - - Assert.Throws(() => builder.ExecuteTestCase()); - } - - [Fact] - public void TestBlockThatFailsThrowsCorrectException() - { - TestBuilder builder = new TestBuilder(); - builder - .AddTestBlock("Bad Value"); - - try - { - builder.ExecuteTestCase(); - } - catch (TestCaseException ex) - { - Assert.Equal(typeof(EqualException), ex.InnerException.GetType()); - } - } - - [Fact] - public void AddLoggerReturnsCorrectLogger() - { - TestBuilder builder = new TestBuilder(); - builder - .AddLogger() - .AddTestBlock(); - - Assert.Throws(() => builder.ExecuteTestCase()); - } - - [Fact] - public void AddFinallyBlockThrowsExpectedException() - { - TestBuilder builder = new TestBuilder(); - builder - .AddTestBlock(false) - .AddFinallyBlock() - .ExecuteTestCase(); - } - - // Actually... this probably shouldn't throw since it's a "finally" block meant to clean stuff up - // Figure out the right behavior and fix test before moving further with finally blocks - [Fact] - public void AddFinallyBlockDoesNotThrowIfExceptionOccursInFinally() - { - TestBuilder builder = new TestBuilder(); - builder - .AddTestBlock(true) - .AddFinallyBlock() - .ExecuteTestCase(); - } - - // How do we verify this is working correctly? - [Fact] - public void AddFinallyBlockExecutesAfterException() - { - TestBuilder builder = new TestBuilder(); - builder - .AddDependencyInstance(true) - .AddTestBlock() - .AddFinallyBlock(); - - Assert.Throws(() => builder.ExecuteTestCase()); - } - - [Fact] - public void OverridingLoggerDoesNotThrow() - { - TestBuilder builder = new TestBuilder(); - builder - .RemoveLogger() - .AddTestBlock("Testing") - .ExecuteTestCase(); - } - - [Fact] - public void RemovingLoggerTwiceDoesNotThrow() - { - TestBuilder builder = new TestBuilder(); - builder - .RemoveLogger() - .RemoveLogger() - .AddTestBlock("Testing") - .ExecuteTestCase(); - } - - [Fact] - public void AddingLoggerThanRemovingDoesNotThrow() - { - TestBuilder builder = new TestBuilder(); - builder - .AddLogger() - .RemoveLogger() - .AddTestBlock("Testing") - .ExecuteTestCase(); - } - - [Fact] - public void OverrideTestCaseNameWithConstructor() - { - TestBuilder builder = new TestBuilder("Testing"); - builder - .AddLogger() - .RemoveLogger() - .AddTestBlock("Testing") - .ExecuteTestCase(); - } - - [Fact] - public void OverrideTestCaseNameWithMethod() - { - TestBuilder builder = new TestBuilder(); - builder - .OverrideTestCaseKey() - .AddLogger() - .RemoveLogger() - .AddTestBlock("Testing") - .ExecuteTestCase(); - } - - [Fact] - public void OverrideTestCaseNameWithMethodOverride() - { - TestBuilder builder = new TestBuilder(); - builder - .OverrideTestCaseKey("Testing") - .AddLogger() - .RemoveLogger() - .AddTestBlock("Testing") - .ExecuteTestCase(); - } - - [Fact] - public void PropertyWithNoMatchingTypeInDiThrowsInvalidOperation() - { - TestBuilder builder = new TestBuilder(); - builder.AddTestBlock(); - - Exception ex = Assert.Throws(() => builder.ExecuteTestCase()); - Assert.Equal(typeof(InvalidOperationException), ex.InnerException.GetType()); - } - } - - public class ExampleTestBlockWithExecuteArg : ITestBlock - { - public void Execute(string input) - { - Assert.Equal("Testing", input); - } - } - - public class ExampleTestBlockWithProperty : ITestBlock - { - public string Input { get; set; } - - public void Execute() - { - Assert.Equal("Testing", Input); - } - } - - public class ExampleTestBlockWithConstructor : ITestBlock - { - public ExampleTestBlockWithConstructor(string input) - { - Input = input; - } - - public void Execute() - { - Assert.Equal("Testing", Input); - } - - private string Input { get; } - } - - public class ExampleTestBlockWithMultipleDependencies : ITestBlock - { - public string InputText { get; set; } - - public void Execute(int inputNumber) - { - Assert.Equal("Testing", InputText); - Assert.Equal(1234, inputNumber); - } - } - - public class ExampleTestBlockWithExecuteArgForOwnType : ITestBlock - { - public void Execute(ExampleDataThing input) - { - if (input == null) throw new ArgumentNullException(nameof(input)); - Assert.Equal("Testing", input.Testing); - } - } - - public class ExampleTestBlockWithExecuteArgForInterface : ITestBlock - { - public void Execute(IExampleDataInterface input) - { - if (input == null) throw new ArgumentNullException(nameof(input)); - Assert.Equal("Testing", input.Testing); - } - } - - public class ExampleTestBlockWithPropertyForOwnType : ITestBlock - { - public ExampleDataThing Input { get; set; } - - public void Execute() - { - Assert.Equal("Testing", Input.Testing); - } - } - - public class ExampleTestBlockWithConstructorForOwnType : ITestBlock - { - public ExampleTestBlockWithConstructorForOwnType(ExampleDataThing input) - { - Input = input; - } - - public void Execute() - { - Assert.Equal("Testing", Input.Testing); - } - - private ExampleDataThing Input { get; } - } - - public class ExampleTestBlockForFactoryWithExecuteArg : ITestBlock - { - public void Execute(ExampleDataThing input) - { - if (input == null) throw new ArgumentNullException(nameof(input)); - Assert.Equal("TestingOverride", input.Testing); - } - } - - public class ExampleTestBlockForFactoryWithProperty : ITestBlock - { - public ExampleDataThing Input { get; set; } - - public void Execute() - { - Assert.Equal("TestingOverride", Input.Testing); - } - } - - public class ExampleTestBlockForFactoryWithConstructor : ITestBlock - { - public ExampleTestBlockForFactoryWithConstructor(ExampleDataThing input) - { - Input = input; - } - - public void Execute() - { - Assert.Equal("TestingOverride", Input.Testing); - } - - private ExampleDataThing Input { get; } - } - - public class ExampleTestBlockWithPropertyWithNoSetter : ITestBlock - { - public string Input { get; } - - public void Execute() - { - Assert.Null(Input); - } - } - - public class ExampleTestBlockWithMultipleExecuteMethods : ITestBlock - { - public void Execute() - { - Assert.True(true); - } - - public void Execute(string input) - { - Assert.Equal("Tetsing", input); - } - } - - public class ExampleLoggerUsage : ITestBlock - { - public void Execute(ILogger log) - { - if (log == null) throw new ArgumentNullException(nameof(log)); - log.Debug("This should throw"); - } - } - - public class ExampleTestBlockWithReturn : ITestBlock - { - public bool Execute(bool valueToReturn) - { - return !valueToReturn; - } - } - - public class ExampleFinallyBlock : ITestBlock - { - public void Execute(bool result) - { - Assert.True(result, "Finally block did not receive correct input"); - } - } - - public class ExampleLogger : ILogger - { - public string TestCaseKey { get; set; } - public string CurrentTestBlock { get; set; } - - public void Debug(string message) - { - throw new NotImplementedException(); - } - - public void Error(string message) - { - throw new NotImplementedException(); - } - - public void Info(string message) - { - throw new NotImplementedException(); - } - - public void TestBlockInput(string input) - { - throw new NotImplementedException(); - } - - public void TestBlockOutput(string output) - { - throw new NotImplementedException(); - } - } -} diff --git a/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework.Tests/TestBuilderTests/ErrorConditions/ExecuteMethodErrors.cs b/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework.Tests/TestBuilderTests/ErrorConditions/ExecuteMethodErrors.cs new file mode 100644 index 00000000..feefabc7 --- /dev/null +++ b/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework.Tests/TestBuilderTests/ErrorConditions/ExecuteMethodErrors.cs @@ -0,0 +1,54 @@ +using System; +using Xunit; + +namespace IntelliTect.TestTools.TestFramework.Tests.TestBuilderTests.ErrorConditions +{ + public class ExecuteMethodErrors + { + [Fact] + public void BuildWithMissingExecuteMethodThrowsInvalidOperationException() + { + // Arrange + TestBuilder builder = new(); + + // Act + var result = Assert.Throws(() => + builder.AddTestBlock()); + + // Assert + Assert.Contains( + ErrorMessages.ExecuteError, + result.Message, + StringComparison.InvariantCultureIgnoreCase); + } + + [Fact] + public void BuildWithTwoExecuteMethodsThrowsInvalidOperationException() + { + // Arrange + TestBuilder builder = new(); + + // Act + var result = Assert.Throws(() => + builder.AddTestBlock()); + + // Assert + Assert.Contains( + ErrorMessages.ExecuteError, + result.Message, + StringComparison.InvariantCultureIgnoreCase); + } + } + + public class ExampleTestBlockWithMultipleExecuteMethods : TestBlock + { + public void Execute() + { + } + + public void Execute(string input) + { + Assert.Equal("Testing", input); + } + } +} diff --git a/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework.Tests/TestBuilderTests/ErrorConditions/ExecuteOverrideTests.cs b/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework.Tests/TestBuilderTests/ErrorConditions/ExecuteOverrideTests.cs new file mode 100644 index 00000000..8446b23e --- /dev/null +++ b/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework.Tests/TestBuilderTests/ErrorConditions/ExecuteOverrideTests.cs @@ -0,0 +1,126 @@ +using IntelliTect.TestTools.TestFramework.Tests.TestData.TestBlocks; +using System; +using System.Collections.Generic; +using System.Text; +using Xunit; + +namespace IntelliTect.TestTools.TestFramework.Tests.TestBuilderTests.ErrorConditions +{ + public class ExecuteOverrideTests : TestBase + { + [Fact] + public void BuildWithMismatchedOverrideAndNoOtherMatchingThrowsAggregateException() + { + // Arrange + TestBuilder builder = new(); + builder.AddTestBlock(1); + + // Act + var result = Assert.Throws(() => + builder.Build()); + + // Assert + ValidateAggregateException( + result, + 2, + ErrorMessages.MissingInputError, + ErrorMessages.MismatchedExecuteOverrideError); + } + + [Fact] + public void BuildWithTooManyOverridesAndNoOtherMatchingThrowsAggregateException() + { + // Arrange + TestBuilder builder = new(); + builder.AddTestBlock("Testing1", true); + + // Act + var result = Assert.Throws(() => + builder.Build()); + + // Assert + ValidateAggregateException( + result, + 2, + ErrorMessages.MissingInputError, + ErrorMessages.TooManyExecuteOverridesError); + } + + [Fact] + public void BuildWithMismatchedOverrideAndOneMatchingThrowsAggregateException() + { + // Arrange + TestBuilder builder = new TestBuilder() + .AddDependencyInstance("Testing") + .AddTestBlock(1); + + // Act + var result = Assert.Throws(() => + builder.Build()); + + // Assert + ValidateAggregateException( + result, + 1, + ErrorMessages.MismatchedExecuteOverrideError); + } + + [Fact] + public void BuildWithTooManyOverridesAndOneMatchingThrowsAggregateException() + { + // Arrange + TestBuilder builder = new TestBuilder() + .AddDependencyInstance("Testing") + .AddTestBlock("Testing1", true); + + // Act + var result = Assert.Throws(() => + builder.Build()); + + // Assert + ValidateAggregateException( + result, + 1, + ErrorMessages.TooManyExecuteOverridesError); + } + + [Fact] + public void BuildWithDuplicateExecuteOverridesThrowsAggregateException() + { + // Arrange + TestBuilder builder = new(); + builder.AddTestBlock("Testing1", "Testing2"); + + // Act + var result = Assert.Throws(() => + builder.Build()); + + // Assert + ValidateAggregateException( + result, + 1, + ErrorMessages.AlreadyAddedError); + } + + [Fact] + public void BuildWithExecuteOverrideOutOfOrderReturnThrowsAggregateException() + { + // Arrange + TestBuilder builder = new(); + builder + .AddTestBlock(true) + .AddTestBlock(); + + // Act + var result = Assert.Throws(() => + builder.Build()); + + // Assert + ValidateAggregateException( + result, + 2, + ErrorMessages.MismatchedExecuteOverrideError, + ErrorMessages.MissingInputError); + } + } +} diff --git a/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework.Tests/TestBuilderTests/ErrorConditions/MultipleDependencyErrors.cs b/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework.Tests/TestBuilderTests/ErrorConditions/MultipleDependencyErrors.cs new file mode 100644 index 00000000..bd37f8a2 --- /dev/null +++ b/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework.Tests/TestBuilderTests/ErrorConditions/MultipleDependencyErrors.cs @@ -0,0 +1,112 @@ +using IntelliTect.TestTools.TestFramework.Tests.TestData.TestBlocks; +using System; +using Xunit; + +namespace IntelliTect.TestTools.TestFramework.Tests.TestBuilderTests.ErrorConditions +{ + public class MultipleDependencyErrors : TestBase + { + [Fact] + public void BuildWithMissingDependencyThrowsAggregateException() + { + // Arrange + TestBuilder builder = new TestBuilder() + .AddTestBlock() + .AddTestBlock(); + + // Act + var result = Assert.Throws(() => + builder.Build()); + + // Assert + ValidateAggregateException( + result, + 2, + ErrorMessages.MissingInputError); + } + + [Fact] + public void BuildWithMismatchedDependencyThrowsAggregateException() + { + // Arrange + TestBuilder builder = new TestBuilder() + .AddDependencyInstance(true) + .AddTestBlock() + .AddTestBlock(); + + // Act + var result = Assert.Throws(() => + builder.Build()); + + // Assert + ValidateAggregateException( + result, + 2, + ErrorMessages.MissingInputError); + } + + [Fact] + public void BuildWithMismatchedDependencyAsTestBlockParamThrowsAggregateException() + { + // Arrange + TestBuilder builder = new(); + builder + .AddTestBlock(true) + .AddTestBlock(true); + + // Act + var result = Assert.Throws(() => + builder.Build()); + + // Assert + ValidateAggregateException( + result, + 4, + ErrorMessages.MissingInputError, + ErrorMessages.MismatchedExecuteOverrideError); + } + + [Fact] + public void BuildWithMismatchedTestBlockReturnThrowsAggregateException() + { + // Arrange + TestBuilder builder = new(); + builder + .AddDependencyInstance(true) + .AddTestBlock() + .AddTestBlock() + .AddTestBlock(); + + // Act + var result = Assert.Throws(() => + builder.Build()); + + // Assert + ValidateAggregateException( + result, + 2, + ErrorMessages.MissingInputError); + } + + [Fact] + public void BuildWithExecuteOverrideAndMismatchedTestBlockReturnThrowsAggregateException() + { + // Arrange + TestBuilder builder = new(); + builder + .AddTestBlock(true) + .AddTestBlock() + .AddTestBlock(); + + // Act + var result = Assert.Throws(() => + builder.Build()); + + // Assert + ValidateAggregateException( + result, + 2, + ErrorMessages.MissingInputError); + } + } +} diff --git a/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework.Tests/TestBuilderTests/ErrorConditions/SingleDependencyErrors.cs b/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework.Tests/TestBuilderTests/ErrorConditions/SingleDependencyErrors.cs new file mode 100644 index 00000000..73fef88b --- /dev/null +++ b/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework.Tests/TestBuilderTests/ErrorConditions/SingleDependencyErrors.cs @@ -0,0 +1,125 @@ +using IntelliTect.TestTools.TestFramework.Tests.TestData.Dependencies; +using IntelliTect.TestTools.TestFramework.Tests.TestData.TestBlocks; +using System; +using Xunit; + +namespace IntelliTect.TestTools.TestFramework.Tests.TestBuilderTests.ErrorConditions +{ + public class SingleDependencyErrors : TestBase + { + // Make sure to check for out of order returns/dependencies + + [Fact] + public void BuildWithMissingDependencyInstanceThrowsAggregateException() + { + // Arrange + TestBuilder builder = new(); + builder.AddTestBlock(); + + // Act + var result = Assert.Throws(() => + builder.Build()); + + // Assert + ValidateAggregateException( + result, + 1, + ErrorMessages.MissingInputError); + } + + [Fact] + public void BuildWithMismatchedDependencyInstanceThrowsAggregateException() + { + // Arrange + TestBuilder builder = new TestBuilder() + .AddDependencyInstance(1) + .AddTestBlock(); + + // Act + var result = Assert.Throws(() => + builder.Build()); + + // Assert + ValidateAggregateException( + result, + 1, + ErrorMessages.MissingInputError); + } + + [Fact] + public void BuildWithMismatchedExecuteReturnThrowsAggregateException() + { + // Arrange + TestBuilder builder = new(); + builder + .AddDependencyInstance(true) + .AddTestBlock() + .AddTestBlock(); + + // Act + var result = Assert.Throws(() => + builder.Build()); + + // Assert + ValidateAggregateException( + result, + 1, + ErrorMessages.MissingInputError); + } + + [Fact] + public void BuildWithOutOfOrderReturnThrowsAggregateException() + { + // Arrange + TestBuilder builder = new(); + builder + .AddDependencyInstance(true) + .AddTestBlock() + .AddTestBlock(); + + // Act + var result = Assert.Throws(() => + builder.Build()); + + // Assert + ValidateAggregateException( + result, + 1, + ErrorMessages.MissingInputError); + } + + [Fact] + public void BuildWithExecuteOverrideMismatchedExecuteReturnThrowsAggregateException() + { + // Arrange + TestBuilder builder = new(); + builder + .AddTestBlock(true) + .AddTestBlock(); + + // Act + var result = Assert.Throws(() => + builder.Build()); + + // Assert + ValidateAggregateException( + result, + 1, + ErrorMessages.MissingInputError); + } + + [Fact] + public void AddNullInstanceAndTypeThrowsArgumentNullException() + { + Assert.Throws(() => new TestBuilder() + .AddDependencyInstance(null!)); + } + + [Fact] + public void AddNullInstanceThrowsArgumentNullException() + { + Assert.Throws(() => new TestBuilder() + .AddDependencyInstance(null!)); + } + } +} diff --git a/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework.Tests/TestBuilderTests/PositiveConditions/LoggerTests.cs b/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework.Tests/TestBuilderTests/PositiveConditions/LoggerTests.cs new file mode 100644 index 00000000..5e4e4531 --- /dev/null +++ b/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework.Tests/TestBuilderTests/PositiveConditions/LoggerTests.cs @@ -0,0 +1,188 @@ +using Xunit; + +namespace IntelliTect.TestTools.TestFramework.Tests.TestBuilderTests +{ + public class LoggerTests + { + [Fact] + public void DefaultLoggerIsAddedOnCreate() + { + // Arrange + TestCase tc = new TestBuilder() + .AddTestBlock() + .Build(); + + // Act / Assert + tc.Execute(); + } + + [Fact] + public void RemovedLoggerDoesNotThrowWhenAttemptingToActivateProp() + { + // Arrange + TestCase tc = new TestBuilder() + .RemoveLogger() + .AddTestBlock() + .Build(); + + // Act / Assert + tc.Execute(); + } + + [Fact] + public void RemovedLoggerDoesNotThrowWhenAttemptingToActivateCtor() + { + // Arrange + TestCase tc = new TestBuilder() + .RemoveLogger() + .AddTestBlock() + .Build(); + + // Act / Assert + tc.Execute(); + } + + [Fact] + public void RemovedLoggerDoesNotThrowWhenAttemptingToActivateExecuteArg() + { + // Arrange + TestCase tc = new TestBuilder() + .RemoveLogger() + .AddTestBlock() + .Build(); + + // Act / Assert + tc.Execute(); + } + + [Fact] + public void CustomLoggerAddsWithoutError() + { + // Arrange + TestCase tc = new TestBuilder() + .AddLogger() + .AddTestBlock() + .Build(); + + // Act / Assert + tc.Execute(); + } + + [Fact] + public void RemovingLoggerTwiceDoesNotThrow() + { + // Arrange + TestCase tc = new TestBuilder() + .RemoveLogger() + .RemoveLogger() + .AddTestBlock() + .Build(); + + // Act / Assert + tc.Execute(); + } + + [Fact] + public void AddingLoggerThanRemovingDoesNotThrow() + { + // Arrange + TestCase tc = new TestBuilder() + .AddLogger() + .RemoveLogger() + .AddTestBlock() + .Build(); + + // Act / Assert + tc.Execute(); + } + } + + public class DefaultLogBlock : TestBlock + { + public void Execute() + { + Assert.NotNull(Log); + Assert.IsType(Log); + } + } + + public class RemovedLogBlockProp : TestBlock + { + public void Execute() + { + Assert.Null(Log); + } + } + + public class RemovedLogBlockCtor : TestBlock + { + public RemovedLogBlockCtor(ITestCaseLogger? log) + { + Log = log; + } + + public void Execute() + { + Assert.Null(Log); + } + } + + public class RemovedLogBlockExecuteArg : TestBlock + { + public void Execute(ITestCaseLogger? log) + { + Assert.Null(log); + } + } + + public class CustomLogBlock : TestBlock + { + public void Execute() + { + Assert.NotNull(Log); + Assert.IsType(Log); + CustomLogger cl = (CustomLogger)Log; + { + Assert.True(cl.Invoked); + } + } + } + + public class CustomLogger : ITestCaseLogger + { + public CustomLogger(TestCase tc) + { + TestCase = tc; + } + public TestCase TestCase { get; } + + public string? CurrentTestBlock { get; set; } + + public bool Invoked { get; set; } + + public void Critical(string message) + { + Invoked = true; + } + + public void Debug(string message) + { + Invoked = true; + } + + public void Info(string message) + { + Invoked = true; + } + + public void TestBlockInput(object input) + { + Invoked = true; + } + + public void TestBlockOutput(object output) + { + Invoked = true; + } + } +} diff --git a/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework.Tests/TestBuilderTests/PositiveConditions/ResolverTests.cs b/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework.Tests/TestBuilderTests/PositiveConditions/ResolverTests.cs new file mode 100644 index 00000000..f5631ed6 --- /dev/null +++ b/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework.Tests/TestBuilderTests/PositiveConditions/ResolverTests.cs @@ -0,0 +1,80 @@ +using IntelliTect.TestTools.TestFramework.Tests.TestData.Dependencies; +using System; +using Xunit; + +namespace IntelliTect.TestTools.TestFramework.Tests.TestBuilderTests.PositiveConditions +{ + public class ResolverTests + { + [Fact] + public void DiReturnsCorrectObjectWhenBlockAsksForInterface() + { + TestCase tc = new TestBuilder() + .AddDependencyService() + .AddTestBlock() + .Build(); + + tc.Execute(); + } + + [Fact] + public void DiReturnsCorrectObjectWhenBlockAsksForImplementation() + { + TestCase tc = new TestBuilder() + .AddDependencyService() + .AddTestBlock() + .Build(); + + tc.Execute(); + } + + [Fact] + public void TestCaseThrowsIfBuildingWithMismatchedImplementation() + { + var tb = new TestBuilder() + .AddDependencyService() + .AddTestBlock(); + + var ex = Assert.Throws(() => tb.Build()); + Assert.Single(ex.InnerExceptions); + } + + [Fact] + public void DiReturnsCorrectObjectWhenBlockAsksOneOfMultipleImplementations() + { + TestCase tc = new TestBuilder() + .AddDependencyService() + .AddDependencyService() + .AddTestBlock() + .Build(); + + tc.Execute(); + } + + // Need to also test a dependency that's implementing multiple interfaces + } + + public class InterfaceBlock : TestBlock + { + public void Execute(IExampleDataInterface iface) + { + Assert.NotNull(iface); + } + } + + public class ImplementationBlock : TestBlock + { + public void Execute(ExampleImplementation iface) + { + Assert.NotNull(iface); + } + } + + public class IncorrectInterfaceBlock : TestBlock + { + public void Execute(OtherExampleImplementation iface) + { + Assert.NotNull(iface); + } + } +} diff --git a/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework.Tests/TestBuilderTests/PositiveConditions/TestCasePropertyTests.cs b/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework.Tests/TestBuilderTests/PositiveConditions/TestCasePropertyTests.cs new file mode 100644 index 00000000..639f0c48 --- /dev/null +++ b/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework.Tests/TestBuilderTests/PositiveConditions/TestCasePropertyTests.cs @@ -0,0 +1,170 @@ +using IntelliTect.TestTools.TestFramework.Tests.TestData.TestBlocks; +using System; +using System.Collections.Generic; +using System.Text; +using Xunit; + +namespace IntelliTect.TestTools.TestFramework.Tests.TestBuilderTests +{ + public class TestCasePropertyTests + { + private const string _Name = "New Name"; + + [Fact] + public void TestCaseNameCanBeChanged() + { + // Arrange / Act + TestCase tc = new TestBuilder() + .AddTestCaseName(_Name) + .Build(); + + // Assert + Assert.Equal(_Name, tc.TestCaseName); + } + + [Fact] + public void NullTestMethodNameIsOverriddenToGenericTestCaseName() + { + // Arrange / Act + TestBuilder tb = new(null); + TestCase tc = tb.Build(); + + // Assert + Assert.Equal("UndefinedTestMethodName", tc.TestCaseName); + } + + [Fact] + public void TestCaseNameIsOverriddenToCallingMethod() + { + // Arrange / Act + TestBuilder tb = new(); + TestCase tc = tb.Build(); + + // Assert + Assert.Equal(nameof(TestCaseNameIsOverriddenToCallingMethod), tc.TestCaseName); + } + + [Fact] + public void TestCaseNameIsOverriddenByConstructorArg() + { + // Arrange / Act + TestBuilder tb = new(_Name); + TestCase tc = tb.Build(); + + // Assert + Assert.Equal(_Name, tc.TestCaseName); + } + + [Fact] + public void NullTestMethodNameIsOverriddenToGenericTestMethodName() + { + // Arrange / Act + TestBuilder tb = new(null); + TestCase tc = tb.Build(); + + // Assert + Assert.Equal("UndefinedTestMethodName", tc.TestMethodName); + } + + [Fact] + public void TestMethodNameIsOverriddenToCallingMethod() + { + // Arrange / Act + TestBuilder tb = new(); + TestCase tc = tb.Build(); + + // Assert + Assert.Equal(nameof(TestMethodNameIsOverriddenToCallingMethod), tc.TestMethodName); + } + + [Fact] + public void TestMethodNameIsOverriddenByConstructorArg() + { + // Arrange / Act + TestBuilder tb = new(_Name); + TestCase tc = tb.Build(); + + // Assert + Assert.Equal(_Name, tc.TestMethodName); + } + + [Fact] + public void TestCaseIdDefaultsToZero() + { + // Arrange / Act + TestBuilder tb = new(); + TestCase tc = tb.Build(); + + // Assert + Assert.Equal(0, tc.TestCaseId); + } + + [Fact] + public void TestCaseIdCanBeOverridden() + { + // Arrange / Act + TestCase tc = new TestBuilder() + .AddTestCaseId(1) + .Build(); + + // Assert + Assert.Equal(1, tc.TestCaseId); + } + + // May not need below test. + // Still undecided if this should even be configurable. + [Fact] + public void ThrowOnFinallyBlockDefaultsToTrue() + { + // Arrange / Act + TestBuilder tb = new(); + TestCase tc = tb.Build(); + + // Assert + Assert.True(tc.ThrowOnFinallyBlockException); + } + + [Fact] + public void TestCasePassedDefaultsToFalse() + { + // Arrange / Act + TestBuilder tb = new(); + TestCase tc = tb.Build(); + + // Assert + Assert.False(tc.Passed); + } + + [Fact] + public void TestCasePassedRemainsFalseOnFailure() + { + // Arrange + TestCase tc = new TestBuilder() + .AddDependencyInstance("Fail") + .AddTestBlock() + .Build(); + + // Act + Assert.Throws(() => tc.Execute()); + + // Assert + Assert.False(tc.Passed); + } + + [Fact] + public void TestCasePassedTurnsTrueOnSuccessfulExecution() + { + // Arrange + TestCase tc = new TestBuilder() + .AddDependencyInstance("Testing") + .AddTestBlock() + .Build(); + + // Act + tc.Execute(); + + // Assert + Assert.True(tc.Passed); + } + } +} diff --git a/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework.Tests/TestCaseTests/ExecuteArgumentOverrides.cs b/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework.Tests/TestCaseTests/ExecuteArgumentOverrides.cs new file mode 100644 index 00000000..b1c0b848 --- /dev/null +++ b/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework.Tests/TestCaseTests/ExecuteArgumentOverrides.cs @@ -0,0 +1,80 @@ +using IntelliTect.TestTools.TestFramework.Tests.TestData.TestBlocks; +using System; +using System.Collections.Generic; +using System.Text; +using Xunit; + +namespace IntelliTect.TestTools.TestFramework.Tests.TestCaseTests +{ + public class ExecuteArgumentOverrides + { + // Test for... + // One execute override successfully overrides something from DI container + // Two different execute overrides successfully inject + + [Fact] + public void ExecuteOverrideInjectsCorrectDependency() + { + // Arrange + TestCase tc = new TestBuilder() + .AddDependencyInstance("This will fail.") + .AddTestBlock("Testing") + .Build(); + + // Act + tc.Execute(); + + // Assert + Assert.True(tc.Passed); + } + + [Fact] + public void ExecuteOverrideWithMultipleDependenciesInjectsCorrectDependency() + { + // Arrange + TestCase tc = new TestBuilder() + .AddDependencyInstance("This will fail.") + .AddDependencyInstance(1234) + .AddTestBlock("Testing") + .Build(); + + // Act + tc.Execute(); + + // Assert + Assert.True(tc.Passed); + } + + [Fact] + public void ExecuteMultipleOverridesWithMultipleDependenciesInjectsCorrectDependency() + { + // Arrange + TestCase tc = new TestBuilder() + .AddDependencyInstance("This will fail.") + .AddDependencyInstance(4321) + .AddTestBlock("Testing", 1234) + .Build(); + + // Act + tc.Execute(); + + // Assert + Assert.True(tc.Passed); + } + + [Fact] + public void ExecuteMultipleOverridesOutOfOrderInjectsCorrectDependency() + { + // Arrange + TestCase tc = new TestBuilder() + .AddTestBlock(1234, "Testing") + .Build(); + + // Act + tc.Execute(); + + // Assert + Assert.True(tc.Passed); + } + } +} diff --git a/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework.Tests/TestCaseTests/FinallyExecutionTests.cs b/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework.Tests/TestCaseTests/FinallyExecutionTests.cs new file mode 100644 index 00000000..9a13f9d2 --- /dev/null +++ b/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework.Tests/TestCaseTests/FinallyExecutionTests.cs @@ -0,0 +1,91 @@ +using IntelliTect.TestTools.TestFramework.Tests.TestData.Dependencies; +using IntelliTect.TestTools.TestFramework.Tests.TestData.TestBlocks; +using System; +using Xunit; + +namespace IntelliTect.TestTools.TestFramework.Tests.TestCaseTests +{ + public class FinallyExecutionTests + { + + [Fact] + public void FinallyBlockThrowsExpectedExceptionWhenNotOverridingDefaultFinallyBehavior() + { + // Arrange + TestCase tc = new TestBuilder() + .AddDependencyInstance(true) + .AddTestBlock() + .AddFinallyBlock() + .Build(); + + // Act + var ex = Assert.Throws(() => tc.Execute()); + + // Assert + Assert.NotNull(ex.InnerExceptions); + Assert.Single(ex.InnerExceptions); + Assert.Contains("Test case succeeded", + ex.Message, + StringComparison.InvariantCultureIgnoreCase); + Assert.True(tc.Passed, "Test case did not get marked as Passed when we expected it."); + } + + [Fact] + public void TestBlockAndFinallyBlockThrowsExpectedExceptionWhenNotOverridingDefaultFinallyBehavior() + { + // Arrange + TestCase tc = new TestBuilder() + .AddDependencyInstance(false) + .AddTestBlock() + .AddFinallyBlock() + .Build(); + + // Act + var ex = Assert.Throws(() => tc.Execute()); + + // Assert + Assert.NotNull(ex.InnerExceptions); + Assert.Equal(2, ex.InnerExceptions.Count); + Assert.Contains("Test case failed and finally blocks failed", + ex.Message, + StringComparison.InvariantCultureIgnoreCase); + Assert.False(tc.Passed, "Test case did not get marked as Failed when we expected it."); + } + + [Fact] + public void FinallyBlockDoesNotThrowExceptionWhenOverridingDefaultFinallyBehavior() + { + // Arrange + TestCase tc = new TestBuilder() + .AddDependencyInstance(true) + .AddTestBlock() + .AddFinallyBlock() + .Build(); + tc.ThrowOnFinallyBlockException = false; + + // Act + tc.Execute(); + + // Assert + Assert.True(tc.Passed, "Test case did not get marked as Passed when we expected it."); + } + + [Fact] + public void OnlyTestBlockThrowsExpectedExceptionWhenOverridingDefaultFinallyBehavior() + { + // Arrange + TestCase tc = new TestBuilder() + .AddDependencyInstance(false) + .AddTestBlock() + .AddFinallyBlock() + .Build(); + tc.ThrowOnFinallyBlockException = false; + + // Act + Assert.Throws(() => tc.Execute()); + + // Assert + Assert.False(tc.Passed, "Test case did not get marked as Failed when we expected it."); + } + } +} diff --git a/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework.Tests/TestCaseTests/MultipleDependencyTests.cs b/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework.Tests/TestCaseTests/MultipleDependencyTests.cs new file mode 100644 index 00000000..56822df7 --- /dev/null +++ b/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework.Tests/TestCaseTests/MultipleDependencyTests.cs @@ -0,0 +1,42 @@ +using IntelliTect.TestTools.TestFramework.Tests.TestData.Dependencies; +using IntelliTect.TestTools.TestFramework.Tests.TestData.TestBlocks; +using Xunit; + +namespace IntelliTect.TestTools.TestFramework.Tests.TestCaseTests +{ + public class MultipleDependencyTests + { + [Fact] + public void ReturnDuplicateTypesDoesNotThrow() + { + // Arrange + TestCase tc = new TestBuilder() + .AddTestBlock(true) + .AddTestBlock(true) + .Build(); + + // Act + tc.Execute(); + + // Assert + Assert.True(tc.Passed); + } + + [Fact] + public void FetchByObjectInstanceForMultipleDependencies() + { + // Arrange + TestCase tc = new TestBuilder() + .AddDependencyInstance("Testing") + .AddDependencyInstance(1234) + .AddTestBlock() + .Build(); + + // Act + tc.Execute(); + + // Assert + Assert.True(tc.Passed); + } + } +} diff --git a/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework.Tests/TestCaseTests/SingleDependencyTests.cs b/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework.Tests/TestCaseTests/SingleDependencyTests.cs new file mode 100644 index 00000000..fd066240 --- /dev/null +++ b/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework.Tests/TestCaseTests/SingleDependencyTests.cs @@ -0,0 +1,146 @@ +using IntelliTect.TestTools.TestFramework.Tests.TestData.Dependencies; +using IntelliTect.TestTools.TestFramework.Tests.TestData.TestBlocks; +using Xunit; + +namespace IntelliTect.TestTools.TestFramework.Tests.TestCaseTests +{ + public class SingleDependencyTests + { + // Test for... + // Adding a dependency that itself does not have a satisfied dependency + // A test block output is successfully used in a subsequent test block + // Do we need to test asking for a null type? + // Do we need to test returning a null type? + + [Fact] + public void ExecuteTestWithAvailableInstanceForExecuteArg() + { + // Arrange + TestCase tc = new TestBuilder() + .AddDependencyInstance("Testing") + .AddTestBlock() + .Build(); + + // Act + tc.Execute(); + + // Assert + Assert.True(tc.Passed); + } + + [Fact] + public void ExecuteTestWithAvailableInstanceForTestBlockProperty() + { + // Arrange + TestCase tc = new TestBuilder() + .AddDependencyInstance("Testing") + .AddTestBlock() + .Build(); + + // Act + tc.Execute(); + + // Assert + Assert.True(tc.Passed); + } + + [Fact] + public void ExecuteTestWithAvailableInstanceForTestBlockConstructor() + { + // Arrange + TestCase tc = new TestBuilder() + .AddDependencyInstance("Testing") + .AddTestBlock() + .Build(); + + // Act + tc.Execute(); + + // Assert + Assert.True(tc.Passed); + } + + [Fact] + public void ExecuteTestBlockWitNonSettablePropertyDoesNotThrow() + { + // Arrange + TestCase tc = new TestBuilder() + .AddDependencyInstance("Testing") + .AddTestBlock() + .Build(); + + // Act + tc.Execute(); + + // Assert + Assert.True(tc.Passed); + } + + // Note: the following tests should be exercising out of the box MS DI functionality. + // The purpose is just to ensure that these methods don't ever accidentally get decoupled from the underlying MS service provider. + // That is also why we aren't extensively testing the same scenarios as above for the AddDependencyService method + [Fact] + public void ExecuteTestWithAvailableServiceForExecuteArg() + { + // Arrange + TestCase tc = new TestBuilder() + .AddDependencyService() + .AddTestBlock() + .Build(); + + // Act + tc.Execute(); + + // Assert + Assert.True(tc.Passed); + } + + [Fact] + public void ExecuteTestWithAvailableGenericArgumentAndInstanceForExecuteArg() + { + // Arrange + TestCase tc = new TestBuilder() + .AddDependencyInstance(new ExampleImplementation()) + .AddTestBlock() + .Build(); + + // Act + tc.Execute(); + + // Assert + Assert.True(tc.Passed); + } + + [Fact] + public void ExecuteTestWithAvailableGenericArgumentsForExecuteArg() + { + // Arrange + TestCase tc = new TestBuilder() + .AddDependencyService() + .AddTestBlock() + .Build(); + + // Act + tc.Execute(); + + // Assert + Assert.True(tc.Passed); + } + + [Fact] + public void ExecuteTestWithAvailableFactoryForExecuteArg() + { + // Arrange + TestCase tc = new TestBuilder() + .AddDependencyService(new ExampleFactory().DoesNotThrow) + .AddTestBlock() + .Build(); + + // Act + tc.Execute(); + + // Assert + Assert.True(tc.Passed); + } + } +} diff --git a/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework.Tests/TestCaseTests/TestFailureTests.cs b/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework.Tests/TestCaseTests/TestFailureTests.cs new file mode 100644 index 00000000..8fa9a0a2 --- /dev/null +++ b/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework.Tests/TestCaseTests/TestFailureTests.cs @@ -0,0 +1,44 @@ +using IntelliTect.TestTools.TestFramework.Tests.TestData.Dependencies; +using IntelliTect.TestTools.TestFramework.Tests.TestData.TestBlocks; +using System; +using Xunit; + +namespace IntelliTect.TestTools.TestFramework.Tests.TestCaseTests +{ + public class TestFailureTests + { + [Fact] + public void TestFailureThrowsImmediatelyWithOriginalException() + { + TestBuilder builder = new(); + TestCase tc = builder + .AddDependencyInstance(false) + .AddDependencyInstance("Testing") + .AddTestBlock() + .AddTestBlock(1) + .Build(); + + var ex = Assert.Throws(() => tc.Execute()); + Assert.False(tc.Passed); + Assert.NotNull(ex.InnerException); + Assert.IsType(ex.InnerException); + Assert.Equal("test failure", ex.InnerException!.Message, ignoreCase: true); + } + + [Fact] + public void DependencyWithMissingDependencyThrowsOriginalError() + { + TestCase tc = new TestBuilder() + .AddDependencyService(new ExampleFactory().Throws) + .AddDependencyService() + .AddTestBlock() + .Build(); + + var ex = Assert.Throws(() => tc.Execute()); + Assert.False(tc.Passed); + Assert.NotNull(ex.InnerException); + Assert.IsType(ex.InnerException); + Assert.Contains("oops", ex.InnerException!.Message, StringComparison.InvariantCultureIgnoreCase); + } + } +} diff --git a/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework.Tests/TestData/Dependencies/ExampleFactory.cs b/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework.Tests/TestData/Dependencies/ExampleFactory.cs new file mode 100644 index 00000000..ba719c64 --- /dev/null +++ b/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework.Tests/TestData/Dependencies/ExampleFactory.cs @@ -0,0 +1,44 @@ +using System; + +namespace IntelliTect.TestTools.TestFramework.Tests.TestData.Dependencies +{ + public class ExampleFactory + { + public ExampleFactory() + { + DoesNotThrow = GetExampleObject; + Throws = GetAlwaysThrow; + } + + public Func Throws { get; } + public Func DoesNotThrow { get; private set; } + + private ExampleImplementation GetExampleObject(IServiceProvider service) + { + return new ExampleImplementation { Testing = "TestingOverride" }; + } + + private AlwaysThrow GetAlwaysThrow(IServiceProvider provider) + { + return new AlwaysThrow(); + } + } + + public class AlwaysThrow + { + public AlwaysThrow() + { + throw new InvalidOperationException("Oops"); + } + } + + public class SomeDependency + { + public SomeDependency(AlwaysThrow alwaysThrow) + { + AlwaysThrow = alwaysThrow; + } + + public AlwaysThrow AlwaysThrow { get; } + } +} diff --git a/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework.Tests/TestData/Dependencies/ExampleInterface.cs b/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework.Tests/TestData/Dependencies/ExampleInterface.cs new file mode 100644 index 00000000..c0e94965 --- /dev/null +++ b/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework.Tests/TestData/Dependencies/ExampleInterface.cs @@ -0,0 +1,17 @@ +namespace IntelliTect.TestTools.TestFramework.Tests.TestData.Dependencies +{ + public interface IExampleDataInterface + { + string Testing { get; set; } + } + + public class ExampleImplementation : IExampleDataInterface + { + public string Testing { get; set; } = "Testing"; + } + + public class OtherExampleImplementation : IExampleDataInterface + { + public string Testing { get; set; } = "Testing"; + } +} diff --git a/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework.Tests/TestData/Dependencies/SimulatorClasses.cs b/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework.Tests/TestData/Dependencies/SimulatorClasses.cs new file mode 100644 index 00000000..304be67e --- /dev/null +++ b/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework.Tests/TestData/Dependencies/SimulatorClasses.cs @@ -0,0 +1,111 @@ +using System; +using Xunit; + +namespace IntelliTect.TestTools.TestFramework.Tests.TestData.Dependencies +{ + // BREAK OUT INTO OWN FILES AS NEEDED + + public class ExampleTestBlockWithExecuteArgForOwnType : TestBlock + { + public void Execute(ExampleImplementation input) + { + if (input is null) throw new ArgumentNullException(nameof(input)); + Assert.Equal("Testing", input.Testing); + } + } + + public class ExampleTestBlockWithExecuteArgForInterface : TestBlock + { + public void Execute(IExampleDataInterface input) + { + if (input is null) throw new ArgumentNullException(nameof(input)); + Assert.Equal("Testing", input.Testing); + } + } + + public class ExampleTestBlockWithPropertyForOwnType : TestBlock + { + public ExampleImplementation Input { get; set; } = new(); + + public void Execute() + { + Assert.Equal("Testing", Input.Testing); + } + } + + public class ExampleTestBlockWithConstructorForOwnType : TestBlock + { + public ExampleTestBlockWithConstructorForOwnType(ExampleImplementation input) + { + Input = input; + } + + public void Execute() + { + Assert.Equal("Testing", Input.Testing); + } + + private ExampleImplementation Input { get; } + } + + public class ExampleTestBlockForFactoryWithExecuteArg : TestBlock + { + public void Execute(ExampleImplementation input) + { + if (input is null) throw new ArgumentNullException(nameof(input)); + Assert.Equal("TestingOverride", input.Testing); + } + } + + public class ExampleTestBlockForFactoryWithProperty : TestBlock + { + public ExampleImplementation Input { get; set; } = new(); + + public void Execute() + { + Assert.Equal("TestingOverride", Input.Testing); + } + } + + public class ExampleTestBlockForFactoryWithConstructor : TestBlock + { + public ExampleTestBlockForFactoryWithConstructor(ExampleImplementation input) + { + Input = input; + } + + public void Execute() + { + Assert.Equal("TestingOverride", Input.Testing); + } + + private ExampleImplementation Input { get; } + } + + public class ExampleTestBlockWithPropertyWithNoSetter : TestBlock + { + public string Input { get; } = "Not Set"; + + public void Execute() + { + Assert.Equal("Not Set", Input); + } + } + + public class ExampleLoggerUsage : TestBlock + { + public void Execute(ITestCaseLogger log) + { + if (log is null) throw new ArgumentNullException(nameof(log)); + log.Debug("This should throw"); + } + } + + public class ExampleFinallyBlock : TestBlock + { + public void Execute(bool result) + { + Assert.True(result, "This is an expected failure."); + } + } +} diff --git a/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework.Tests/TestData/TestBlocks/MultipleDependencyBlocks.cs b/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework.Tests/TestData/TestBlocks/MultipleDependencyBlocks.cs new file mode 100644 index 00000000..21dc1ba3 --- /dev/null +++ b/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework.Tests/TestData/TestBlocks/MultipleDependencyBlocks.cs @@ -0,0 +1,24 @@ +using Xunit; + +namespace IntelliTect.TestTools.TestFramework.Tests.TestData.TestBlocks +{ + public class ExampleTestBlockWithMultipleDependencies : TestBlock + { + public string? InputText { get; set; } + + public void Execute(int inputNumber) + { + Assert.Equal("Testing", InputText); + Assert.Equal(1234, inputNumber); + } + } + + public class ExampleTestBlockWithMultipleExecuteArgs : TestBlock + { + public void Execute(string inputText, int inputNumber) + { + Assert.Equal("Testing", inputText); + Assert.Equal(1234, inputNumber); + } + } +} diff --git a/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework.Tests/TestData/TestBlocks/SingleDependencyBlocks.cs b/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework.Tests/TestData/TestBlocks/SingleDependencyBlocks.cs new file mode 100644 index 00000000..b26c1c8d --- /dev/null +++ b/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework.Tests/TestData/TestBlocks/SingleDependencyBlocks.cs @@ -0,0 +1,73 @@ +using IntelliTect.TestTools.TestFramework.Tests.TestData.Dependencies; +using System; +using Xunit; + +namespace IntelliTect.TestTools.TestFramework.Tests.TestData.TestBlocks +{ + public class ExampleTestBlockWithExecuteArg : TestBlock + { + public void Execute(string input) + { + Assert.Equal("Testing", input); + } + } + + public class ExampleTestBlockWithProperty : TestBlock + { + public string? Input { get; set; } + + public void Execute() + { + Assert.Equal("Testing", Input); + } + } + + public class ExampleTestBlockWithConstructor : TestBlock + { + public ExampleTestBlockWithConstructor(string input) + { + Input = input; + } + + private string Input { get; } + + public void Execute() + { + Assert.Equal("Testing", Input); + } + } + + public class ExampleTestBlockWithStringReturn : TestBlock + { + public string Execute() + { + return "Testing"; + } + } + + public class ExampleTestBlockWithBoolReturn : TestBlock + { + public bool Execute(bool arg) + { + if (!arg) + { + throw new DivideByZeroException("Test failure"); + } + else + { + return false; + } + } + } + + public class SomeTestBlock : ITestBlock + { + public ITestCaseLogger? Log { get; } + + public bool Execute(SomeDependency dep) + { + Assert.NotNull(dep); + return true; + } + } +} diff --git a/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework/Block.cs b/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework/Block.cs new file mode 100644 index 00000000..c0b3d2ff --- /dev/null +++ b/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework/Block.cs @@ -0,0 +1,23 @@ +using System; +using System.Collections.Generic; +using System.Reflection; + +namespace IntelliTect.TestTools.TestFramework +{ + internal class Block + { + public Block(Type type, MethodInfo execute) + { + Type = type; + ExecuteMethod = execute; + } + + internal Type Type { get; set; } + internal bool IsFinallyBlock { get; set; } + internal ParameterInfo[] ConstructorParams { get; set; } = Array.Empty(); + internal MethodInfo ExecuteMethod { get; set; } + internal ParameterInfo[] ExecuteParams { get; set; } = Array.Empty(); + internal Dictionary ExecuteArgumentOverrides { get; set; } = new(); + internal PropertyInfo[] PropertyParams { get; set; } = Array.Empty(); + } +} diff --git a/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework/DebugLogger.cs b/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework/DebugLogger.cs new file mode 100644 index 00000000..7f640d37 --- /dev/null +++ b/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework/DebugLogger.cs @@ -0,0 +1,68 @@ +using System; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace IntelliTect.TestTools.TestFramework +{ + public class DebugLogger : ITestCaseLogger + { + public DebugLogger(TestCase testCase) + { + TestCase = testCase; + } + + public TestCase TestCase { get; } + public string? CurrentTestBlock { get; set; } + + public void Debug(string message) + { + LogToDebug($"{TestCase.TestMethodName} - {CurrentTestBlock} - Debug: {message}"); + } + + public void Critical(string message) + { + LogToDebug($"{TestCase.TestMethodName} - {CurrentTestBlock} - Error: {message}"); + } + + public void Info(string message) + { + LogToDebug($"{TestCase.TestMethodName} - {CurrentTestBlock} - Info: {message}"); + } + + public void TestBlockInput(object input) + { + string inputString = Serialize(input); + LogToDebug($"{TestCase.TestMethodName} - {CurrentTestBlock} - Input arguments: {inputString}"); + } + + public void TestBlockOutput(object output) + { + string outputString = Serialize(output); + LogToDebug($"{TestCase.TestMethodName} - {CurrentTestBlock} - Output returns: {outputString}"); + } + + private void LogToDebug(object message) + { + System.Diagnostics.Debug.WriteLine(message); + } + + private string Serialize(object objectToParse) + { + if(objectToParse is null) throw new ArgumentNullException(nameof(objectToParse)); + // JsonSerializer.Serialize has some different throw behavior between versions. + // One version threw an exception that occurred on a property, which happened to be a Selenium WebDriverException. + // In this one specific case, catch all exceptions and move on to provide standard behavior to all package consumers. + // TL;DR: we don't want logging failures to interrupt the test run. + try + { + return JsonSerializer.Serialize(objectToParse, new JsonSerializerOptions { WriteIndented = true }); + } +#pragma warning disable CA1031 // Do not catch general exception types + catch (Exception e) +#pragma warning restore CA1031 // Do not catch general exception types + { + return $"Unable to serialize object {objectToParse.GetType()} to JSON. Mark the relevant property with the [{nameof(JsonIgnoreAttribute)}] attribute: {e}"; + } + } + } +} diff --git a/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework/ILogger.cs b/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework/ILogger.cs deleted file mode 100644 index 935dbb55..00000000 --- a/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework/ILogger.cs +++ /dev/null @@ -1,20 +0,0 @@ -namespace IntelliTect.TestTools.TestFramework -{ - public interface ILogger - { - // Probably need to handle this differently - // Maybe a constructor sets the test case name? - string TestCaseKey { get; set; } - string CurrentTestBlock { get; set; } - void Debug(string message); - void Info(string message); - - [System.Diagnostics.CodeAnalysis.SuppressMessage( - "Naming", - "CA1716:Identifiers should not match keywords", - Justification = "Deferring to next major rev.")] - void Error(string message); - void TestBlockInput(string input); - void TestBlockOutput(string output); - } -} diff --git a/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework/ITestBlock.cs b/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework/ITestBlock.cs index 303b9a59..dc386d4f 100644 --- a/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework/ITestBlock.cs +++ b/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework/ITestBlock.cs @@ -1,11 +1,7 @@ namespace IntelliTect.TestTools.TestFramework { - [System.Diagnostics.CodeAnalysis.SuppressMessage( - "Design", - "CA1040:Avoid empty interfaces", - Justification = "Deferring to next major rev")] public interface ITestBlock { - + ITestCaseLogger? Log { get; } } -} +} \ No newline at end of file diff --git a/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework/ITestCaseLogger.cs b/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework/ITestCaseLogger.cs new file mode 100644 index 00000000..d3c72dd2 --- /dev/null +++ b/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework/ITestCaseLogger.cs @@ -0,0 +1,13 @@ +namespace IntelliTect.TestTools.TestFramework +{ + public interface ITestCaseLogger + { + TestCase TestCase { get; } + string? CurrentTestBlock { get; set; } + void Debug(string message); + void Info(string message); + void Critical(string message); + void TestBlockInput(object input); + void TestBlockOutput(object output); + } +} diff --git a/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework.csproj b/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework.csproj index f0ec9762..12071ac1 100644 --- a/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework.csproj +++ b/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework.csproj @@ -2,18 +2,24 @@ netstandard2.0 - true - 4 - 1701;1702;CA1303 + 11.0 + enable + + CA1303; + all - - - + + + - + + + + + \ No newline at end of file diff --git a/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework/Log.cs b/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework/Log.cs deleted file mode 100644 index 5a91202e..00000000 --- a/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework/Log.cs +++ /dev/null @@ -1,38 +0,0 @@ -namespace IntelliTect.TestTools.TestFramework -{ - public class Log : ILogger - { - public string TestCaseKey { get; set; } - public string CurrentTestBlock { get; set; } - - public void Debug(string message) - { - LogToDebug($"{TestCaseKey} - {CurrentTestBlock} - Debug: {message}"); - } - - public void Error(string message) - { - LogToDebug($"{TestCaseKey} - {CurrentTestBlock} - Error: {message}"); - } - - public void Info(string message) - { - LogToDebug($"{TestCaseKey} - {CurrentTestBlock} - Info: {message}"); - } - - public void TestBlockInput(string input) - { - LogToDebug($"{TestCaseKey} - {CurrentTestBlock} - Input arguments: {input}"); - } - - public void TestBlockOutput(string output) - { - LogToDebug($"{TestCaseKey} - {CurrentTestBlock} - Output returns: {output}"); - } - - private void LogToDebug(string message) - { - System.Diagnostics.Debug.WriteLine(message); - } - } -} diff --git a/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework/TestBlock.cs b/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework/TestBlock.cs new file mode 100644 index 00000000..99906341 --- /dev/null +++ b/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework/TestBlock.cs @@ -0,0 +1,7 @@ +namespace IntelliTect.TestTools.TestFramework +{ + public class TestBlock : ITestBlock + { + public ITestCaseLogger? Log { get; set; } + } +} \ No newline at end of file diff --git a/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework/TestBuilder.cs b/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework/TestBuilder.cs index 58717f77..ed9ed9f8 100644 --- a/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework/TestBuilder.cs +++ b/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework/TestBuilder.cs @@ -1,8 +1,6 @@ using Microsoft.Extensions.DependencyInjection; -using System.Text.Json; using System; using System.Collections.Generic; -using System.Diagnostics; using System.Linq; using System.Reflection; using System.Runtime.CompilerServices; @@ -11,358 +9,319 @@ namespace IntelliTect.TestTools.TestFramework { public class TestBuilder { - public TestBuilder([CallerMemberName]string testCaseKey = null) + /// + /// Constructs a TestBuilder instance with a given test name. + /// + /// The name for the test method, defaults to the calling member. + public TestBuilder([CallerMemberName] string? testMethodName = null) { - TestCaseName = testCaseKey; - AddLogger(); + TestMethodName = testMethodName ?? "UndefinedTestMethodName"; + AddLogger(); } - public TestBuilder OverrideTestCaseKey([CallerMemberName]string testCaseKey = null) + private string? TestCaseName { get; set; } + private string TestMethodName { get; set; } + private int TestCaseId { get; set; } + private List TestBlocks { get; } = new(); + private List FinallyBlocks { get; } = new(); + private IServiceCollection Services { get; } = new ServiceCollection(); + private bool HasLogger { get; set; } = true; + private List ValidationExceptions { get; } = new(); + + /// + /// Used when a test case may be associated to a unique ID. + /// + /// The key associated to this test. + /// + public TestBuilder AddTestCaseId(int testCaseKey) { - TestCaseName = testCaseKey; + TestCaseId = testCaseKey; return this; } /// - /// Adds a test block (some related group of test actions) to the list of blocks to run for any given test case + /// Used to give a friendly name to the test case. Defaults to the test method name. /// - /// The type of test block, as an ITestBlock, to run - /// This - public TestBuilder AddTestBlock() where T : ITestBlock + /// A friendly name associated to the test case. + /// + public TestBuilder AddTestCaseName(string testCaseName) { - TestBlocksAndParams.Add((TestBlockType: typeof(T), TestBlockParameters: null)); - Services.AddTransient(typeof(T)); + TestCaseName = testCaseName; return this; } /// - /// Adds a test block (some related group of test actions) with a list of arguments - /// that must match the associated TestBlock.Execute() method to the list of blocks to run for any given test case + /// Adds a test block (some related group of test actions) with an optional list of arguments.
+ /// Any argument passed here will override all other matched arguments for the blocks TestBlock.Execute() method. /// - /// The type of dependency a test block needs to execute - /// The list of arguments to fulfill a set of Execute(params object[]) parameters + /// The type of dependency a test block needs to execute. + /// The list of arguments to fulfill a set of Execute(params object[]) parameters. /// This public TestBuilder AddTestBlock(params object[] testBlockArgs) where T : ITestBlock { - TestBlocksAndParams.Add((TestBlockType: typeof(T), TestBlockParameters: testBlockArgs)); - Services.AddTransient(typeof(T)); + Block tb = CreateBlock(false, testBlockArgs); + TestBlocks.Add(tb); return this; } - public TestBuilder AddFinallyBlock(params object[] testBlockArgs) where T : ITestBlock + /// + /// Adds a finally block, a special test block that will always run after all test blocks, regardless of if a prior test block fails. + /// + /// The type of dependency a test block needs to execute. + /// The list of arguments to fulfill a set of Execute(params object[]) parameters. + /// + public TestBuilder AddFinallyBlock(params object[] finallyBlockArgs) where T : ITestBlock { - FinallyBlocksAndParams.Add((TestBlockType: typeof(T), TestBlockParameters: testBlockArgs)); - Services.AddTransient(typeof(T)); + Block fb = CreateBlock(true, finallyBlockArgs); + FinallyBlocks.Add(fb); return this; } /// - /// Adds a service as a factory a container that is used to fulfill TestBlock dependencies + /// Adds a service as a factory a container that is used to fulfill TestBlock dependencies. /// - /// The type of dependency a test block needs to execute - /// The factory to provide an instance of the type needed for a test block to execute + /// The type of dependency a test block needs to execute. + /// The factory to provide an instance of the type needed for a test block to execute. /// public TestBuilder AddDependencyService(Func serviceFactory) { - Services.AddScoped(typeof(T), serviceFactory); + Services.AddSingleton(typeof(T), serviceFactory); return this; } /// - /// Adds a service as a Type to the container that is used to fulfill TestBlock dependencies + /// Adds a service as a Type to the container that is used to fulfill TestBlock dependencies. /// /// The type of test block, as an ITestBlock, to run /// This public TestBuilder AddDependencyService() { - Services.AddScoped(typeof(T)); + Services.AddSingleton(typeof(T)); return this; } - public TestBuilder AddDependencyService() + /// + /// Adds a service as a Type with an Implementation that is used to fulfill TestBlock dependencies. + /// + /// The type of the service to add. + /// A specific implementation of the type. + /// This + public TestBuilder AddDependencyService() { - Services.AddScoped(typeof(TServiceType), typeof(TImplementationType)); + Services.AddSingleton(typeof(TServiceType), typeof(TImplementationType)); return this; } /// - /// Adds an instance of a Type to the container that is needed for a TestBlock to execute + /// Adds an instance of a Type to the container that is needed for a TestBlock to execute. /// /// The instance of a Type that a TestBlock needs - /// This + /// this public TestBuilder AddDependencyInstance(object objToAdd) { if (objToAdd is null) throw new ArgumentNullException(nameof(objToAdd)); - // Need to add some testing around this to see if it behaves in a similarly odd fashion as AddLogger when running tests in parallel Services.AddSingleton(objToAdd.GetType(), objToAdd); return this; } + /// + /// Adds an instance of a Type to the container that is needed for a TestBlock to execute. + /// + /// The type of the object. + /// The object to add. + /// public TestBuilder AddDependencyInstance(object objToAdd) { + if (objToAdd is null) throw new ArgumentNullException(nameof(objToAdd)); Services.AddSingleton(typeof(T), objToAdd); return this; } - // Are there other cases where we'll need to add something at this level? - // If so, this shouldn't be called "AddLogger". - // Might need to make this scoped. It's behaving oddly when running tests in parallel - // But only on the "Starting test case" call - public TestBuilder AddLogger() where T : ILogger + /// + /// Adds a new logger to be used during test execution. This will remove any existing loggers. + /// + /// Type of logger. + /// + public TestBuilder AddLogger() where T : ITestCaseLogger { RemoveLogger(); - Services.AddSingleton(typeof(ILogger), typeof(T)); + Services.AddSingleton(typeof(ITestCaseLogger), typeof(T)); + HasLogger = true; return this; } + /// + /// Removes the current logger from the test case dependency registration.
+ /// NOTE: if you remove the logger but have a test block or dependency that needs it, an error will occur. + /// + /// this public TestBuilder RemoveLogger() { - var logger = Services.FirstOrDefault(d => d.ServiceType == typeof(ILogger)); - Services.Remove(logger); + ServiceDescriptor? logger = Services.FirstOrDefault(d => d.ServiceType == typeof(ITestCaseLogger)); + if (logger is { }) Services.Remove(logger); + HasLogger = false; return this; } - public void ExecuteTestCase() + /// + /// Builds the test case. This will validate that test block dependencies are satisfied. + /// + /// An object that can be used to execute a test case. + /// + public TestCase Build() { - #region move to a Build() method and validate all dependencies are satisfied? - var serviceProvider = Services.BuildServiceProvider(); - #endregion - - using (var testCaseScope = serviceProvider.CreateScope()) + if (string.IsNullOrWhiteSpace(TestCaseName)) { - var logger = testCaseScope.ServiceProvider.GetService(); - if (logger != null) - { - logger.TestCaseKey = TestCaseName; - logger.CurrentTestBlock = "N/A"; - } - - logger?.Info("Starting test case."); - - using (var testBlockScope = serviceProvider.CreateScope()) - { - foreach (var tb in TestBlocksAndParams) - { - if (logger != null) logger.CurrentTestBlock = tb.TestBlockType.ToString(); - // Might be more concise to have these as out method parameters instead of if statements after every one - var testBlockInstance = GetTestBlock(testBlockScope, tb.TestBlockType); - if (TestBlockException != null) break; - - SetTestBlockProperties(testBlockScope, testBlockInstance, logger); - if (TestBlockException != null) break; - - MethodInfo execute = GetExecuteMethod(testBlockInstance); - if (TestBlockException != null) break; + TestCaseName = TestMethodName; + } - var executeArgs = GatherTestBlockArguments(testBlockScope, execute, tb); - if (TestBlockException != null) break; + TestCase testCase = new(TestCaseName!, TestMethodName, TestCaseId, Services); + testCase.HasLogger = HasLogger; + Services.AddSingleton(testCase); - RunTestBlocks(testBlockInstance, execute, executeArgs, logger); - if (TestBlockException != null) break; - } + // Probably need to profile all of this for performance at some point. + // Need to make sure if we're running hundreds or thousands of tests that we're not adding significant amount of time to that. - // Need a much better way to handle Finally exceptions... - Exception tempException = TestBlockException; - TestBlockException = null; - // Extract loop above since it's basically the same for finally blocks? - foreach (var fb in FinallyBlocksAndParams) - { - if (logger != null) logger.CurrentTestBlock = fb.TestBlockType.ToString(); - // Might be more concise to have these as out method parameters instead of if statements after every one - // Also these specific ones should not be overwriting TestBlockException - var testBlockInstance = GetTestBlock(testBlockScope, fb.TestBlockType); - if (TestBlockException != null) break; + List outputs = new(); + foreach (Block tb in TestBlocks) + { + GatherDependencies(tb, outputs); + testCase.TestBlocks.Add(tb); + } - SetTestBlockProperties(testBlockScope, testBlockInstance, logger); - if (TestBlockException != null) break; + foreach (Block fb in FinallyBlocks) + { + GatherDependencies(fb, outputs); + testCase.FinallyBlocks.Add(fb); + } - MethodInfo execute = GetExecuteMethod(testBlockInstance); - if (TestBlockException != null) break; + if (ValidationExceptions.Count > 0) + { + throw new AggregateException(ValidationExceptions); + } - var executeArgs = GatherTestBlockArguments(testBlockScope, execute, fb); - if (TestBlockException != null) break; + return testCase; + } - RunTestBlocks(testBlockInstance, execute, executeArgs, logger); - if (TestBlockException != null) break; - } - TestBlockException = tempException; - } + private Block CreateBlock(bool isFinally, params object[] args) + { + Services.AddTransient(typeof(T)); - if(TestBlockException == null) + MethodInfo execute = FindExecuteMethod(typeof(T)); + Block b = new(typeof(T), execute); + foreach (object a in args) + { + if(b.ExecuteArgumentOverrides.ContainsKey(a.GetType())) { - logger?.Info("Test case finished successfully."); + ValidationExceptions.Add(new ArgumentException($"TestBlock: {typeof(T)} - Multiple execute argument overrides of the same type are not allowed: {a.GetType()}")); } else { - logger?.Error($"Test case failed: {TestBlockException}"); + b.ExecuteArgumentOverrides.Add(a.GetType(), a); } } - - serviceProvider.Dispose(); - - if (TestBlockException != null) - { - throw new TestCaseException("Test case failed.", TestBlockException); - } + b.IsFinallyBlock = isFinally; + return b; } - private static string GetObjectDataAsJsonString(object obj) + private static MethodInfo FindExecuteMethod(Type type) { - // JsonSerializer.Serialize has some different throw behavior between versions. - // One version threw an exception that occurred on a property, which happened to be a Selenium WebDriverException. - // In this one specific case, catch all exceptions and move on to provide standard behavior to all package consumers. - // TL;DR: we don't want logging failures to interrupt the test run. - try - { - return JsonSerializer.Serialize(obj, new JsonSerializerOptions { WriteIndented = true }); - } -#pragma warning disable CA1031 // Do not catch general exception types - catch (Exception e) -#pragma warning restore CA1031 // Do not catch general exception types + List? executeMethod = type.GetMethods().Where(m => m.Name.ToUpperInvariant() == "EXECUTE").ToList(); + if (executeMethod.Count is not 1) { - return $"Unable to serialize object {obj?.GetType()} to JSON. Mark the relevant property with the [JsonIgnore] attribute: {e}"; + // Don't add to validation messages we don't have any reasonable assurance what the dependencies should be. + throw new InvalidOperationException( + $"TestBlock: {type} - There must be one and only one Execute method on a test block."); } - + + return executeMethod[0]; } - private object GetTestBlock(IServiceScope scope, Type tbType) + private void GatherDependencies( + Block tb, + List outputs) { - var tb = scope.ServiceProvider.GetService(tbType); - if(tb == null) + ConstructorInfo[]? constructors = tb.Type.GetConstructors(); + + if (constructors.Length > 1) { - TestBlockException = new InvalidOperationException($"Unable to find test block: {tbType.FullName}."); + // Don't add to validation messages we don't have any reasonable assurance what the dependencies should be. + throw new InvalidOperationException( + $"TestBlock: {tb.Type} - TestFramework supports zero or one constructors on test blocks."); } - - return tb; - } - private void SetTestBlockProperties(IServiceScope scope, object testBlockInstance, ILogger logger) - { - // Populate all of our properties - var properties = testBlockInstance.GetType().GetProperties(); - foreach (var prop in properties) + tb.ConstructorParams = constructors[0].GetParameters(); + tb.PropertyParams = tb.Type.GetProperties(); + tb.ExecuteParams = tb.ExecuteMethod.GetParameters(); + // Currently do not support Fields. Should we check for them anyway at least to throw? + + HashSet inputs = new(); + foreach (var c in tb.ConstructorParams) { - if (!prop.CanWrite) - { - logger?.Debug($"Skipping property {prop}. No setter found."); - continue; - } - object propertyValue = scope.ServiceProvider.GetService(prop.PropertyType); - if(propertyValue == null) + inputs.Add(c.ParameterType); + } + foreach (var p in tb.PropertyParams) + { + if(p.CanWrite) { - TestBlockException = new InvalidOperationException($"Unable to find an object or service for property {prop.Name} of type {prop.PropertyType.FullName} on test block {testBlockInstance.GetType()}."); - break; + inputs.Add(p.PropertyType); } - - prop.SetValue(testBlockInstance, propertyValue); } - } - - private MethodInfo GetExecuteMethod(object testBlockInstance) - { - List methods = testBlockInstance.GetType().GetMethods().Where(m => m.Name.ToUpperInvariant() == "EXECUTE").ToList(); - if (methods.Count != 1) + foreach (var e in tb.ExecuteParams) { - TestBlockException = new InvalidOperationException($"There can be one and only one Execute method on a test block. " + - $"Please review test block {testBlockInstance.GetType()}."); - return null; + inputs.Add(e.ParameterType); } - return methods[0]; - } - - private object[] GatherTestBlockArguments(IServiceScope scope, MethodInfo execute, (Type TestBlockType, object[] TestBlockParameters) tb) - { - var executeParams = execute.GetParameters(); - - object[] executeArgs = new object[executeParams.Length]; - - // Is this the right order of checking? Or should we prioritize test block results first? - // Initial thought is that if someone is passing in explicit arguments, they probably have a good reason, so we should start there - // Populate and log all of our Execute arguments - if (executeArgs.Length > 0) + if (!HasLogger) inputs.RemoveWhere(i => i == typeof(ITestCaseLogger)); + if (tb.ExecuteArgumentOverrides.Count is not 0) { - if (tb.TestBlockParameters != null && executeParams.Length == tb.TestBlockParameters.Length) + if (tb.ExecuteArgumentOverrides.Count > tb.ExecuteParams.Length) { - // Eventually need to add more validation around making sure the types match here. - executeArgs = tb.TestBlockParameters; + ValidationExceptions.Add(new ArgumentException($"TestBlock: {tb.Type} - Too many execute overrides were provided. More were handed in than parameters on Execute method.")); } else { - for (int i = 0; i < executeArgs.Length; i++) + foreach (KeyValuePair eao in tb.ExecuteArgumentOverrides) { - object foundResult = TestBlockResults.FirstOrDefault(tbr => tbr.GetType() == executeParams[i].ParameterType) - ?? scope.ServiceProvider.GetService(executeParams[i].ParameterType); - if(foundResult == null) + if (!tb.ExecuteParams.Any(ep => ep.ParameterType == eao.Key)) { - TestBlockException = new InvalidOperationException($"Unable to find an object or service for Execute parameter {executeParams[i].Name} of type {executeParams[i].ParameterType.FullName} on test block {tb.TestBlockType.FullName}."); - break; + ValidationExceptions.Add(new ArgumentException($"TestBlock: {tb.Type} - Unable to find corresponding Execute parameter for override argument {eao}")); + } + else + { + // Input is satisfied by execute argument override. + // No need to check later. + inputs.Remove(eao.Key); } - - executeArgs[i] = foundResult; } } - - // Instead of doing this, might be worth extracting the above for loop into a private method and if that fails, then break out of the foreach we're in now - if (TestBlockException != null) return null; } - return executeArgs; - } - - private void RunTestBlocks(object testBlockInstance, MethodInfo execute, object[] executeArgs, ILogger logger) - { - logger?.Debug($"Starting test block."); - // Log ALL inputs - // Is it worth distinguishing between Properties and Execute args? - PropertyInfo[] props = testBlockInstance.GetType().GetProperties(BindingFlags.NonPublic | BindingFlags.Instance); - object[] allArgs = new object[props.Length + executeArgs.Length]; - for(int i = 0; i < props.Length; i++) + foreach (Type i in inputs) { - allArgs[i] = props[i].GetValue(testBlockInstance); + CheckContainerForFirstLevelDependency(i, outputs, $"TestBlock: {tb.Type} - Unable to satisfy test block input: {i}."); } - - executeArgs.CopyTo(allArgs, props.Length); - foreach (var arg in allArgs) + + Type executeReturns = tb.ExecuteMethod.ReturnType; + if (executeReturns != typeof(void)) { - logger?.TestBlockInput(GetObjectDataAsJsonString(arg)); + outputs.Add(executeReturns); } + } - try + private void CheckContainerForFirstLevelDependency(Type type, List outputs, string errorMessage) + { + ServiceDescriptor? obj = Services.FirstOrDefault(x => x.ServiceType == type || x.ImplementationType == type); + if (obj is null) { - var result = execute.Invoke(testBlockInstance, executeArgs); - if (result != null) + Type? output = outputs.FirstOrDefault(o => o == type || o == type); + + if (output is null) { - logger?.TestBlockOutput(GetObjectDataAsJsonString(result)); - TestBlockResults.Add(result); + if (type is ITestCaseLogger) return; + ValidationExceptions.Add(new InvalidOperationException(errorMessage)); } - - } - catch (TargetInvocationException ex) - { - TestBlockException = ex.InnerException; - return; - } - catch (ArgumentException ex) - { - TestBlockException = ex; - return; - } - catch (TargetParameterCountException ex) - { - ex.Data.Add("AdditionalInfo", "Test block failed: Mismatched count between Execute method arguments and supplied dependencies."); - TestBlockException = ex; - return; } - - logger?.Debug($"Test block completed successfully."); } - - private List<(Type TestBlockType, object[] TestBlockParameters)> TestBlocksAndParams { get; } = new List<(Type TestBlockType, object[] TestBlockParameters)>(); - private List<(Type TestBlockType, object[] TestBlockParameters)> FinallyBlocksAndParams { get; } = new List<(Type TestBlockType, object[] TestBlockParameters)>(); - private IServiceCollection Services { get; } = new ServiceCollection(); - private HashSet TestBlockResults { get; } = new HashSet(); - private string TestCaseName { get; set; } - private Exception TestBlockException { get; set; } } } diff --git a/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework/TestCase.cs b/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework/TestCase.cs new file mode 100644 index 00000000..ee3f18ce --- /dev/null +++ b/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework/TestCase.cs @@ -0,0 +1,432 @@ +using Microsoft.Extensions.DependencyInjection; +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Reflection; + +namespace IntelliTect.TestTools.TestFramework +{ + public class TestCase + { + public TestCase(string testCaseName, string testMethodName, int testCaseId, IServiceCollection services) + { + TestCaseName = testCaseName; + TestMethodName = testMethodName; + TestCaseId = testCaseId; + ServiceCollection = services; + } + // Switch to get; init; when this can be updated to .net5 + // Maybe target .net5 support for v3? + /// + /// The friendly name for the test case. + /// + public string TestCaseName { get; } + /// + /// The unit test method name. Defaults to the calling member for new TestBuilder(). + /// + public string TestMethodName { get; } + /// + /// Any ID associated with the test case, otherwise 0. + /// + public int TestCaseId { get; } + /// + /// If this test case should throw if a finally block has an exception. Defaults to true. + ///
+ /// If a finally block fails and this property is true, the test case is still considered passed internally, but most unit test frameworks will mark the test failed. + /// + public bool ThrowOnFinallyBlockException { get; set; } = true; + + // May make sense to make some of the below public if it's needed for debugging. + // If so, definitely need to change them to internal or private sets. + + internal List TestBlocks { get; set; } = new(); + internal List FinallyBlocks { get; set; } = new(); + internal bool HasLogger { get; set; } = true; + + private ITestCaseLogger? Log { get; set; } + private IServiceCollection ServiceCollection { get; } + private Dictionary BlockOutput { get; } = new(); + private Exception? TestBlockException { get; set; } + private List FinallyBlockExceptions { get; } = new(); + + // Has this test case passed? Will only be true if every regular test block succeeds. + public bool Passed { get; set; } + + /// + /// Executes the test case. + /// + /// The exception describing a test failure. + /// Occurs when finally blocks fail, or the test fails and at least one finally block fails. + public void Execute() + { + ServiceProvider services = ServiceCollection.BuildServiceProvider(); + using (var testCaseScope = services.CreateScope()) + { + Log = testCaseScope.ServiceProvider.GetService(); + if (Log is not null) + { + Log.CurrentTestBlock = "N/A"; + } + + Log?.Info($"Starting test case: {TestCaseName}"); + + foreach (var tb in TestBlocks) + { + if (Log is not null) Log.CurrentTestBlock = tb.Type.ToString(); + Log?.Debug($"Starting test block: {tb.Type}"); + + if (!TryGetBlock(testCaseScope, tb, out object testBlockInstance)) break; + if (!TrySetBlockProperties(testCaseScope, tb, testBlockInstance)) break; + if (!TryGetExecuteArguments(testCaseScope, tb, out List executeArgs)) break; + + if (!TryRunBlock(tb, testBlockInstance, executeArgs)) + { + if (TestBlockException is null) + { + TestBlockException = new( + $"Unknown error occurred while running test block {tb}. " + + "Please file an issue: https://github.com/IntelliTect/TestTools/issues"); + } + break; + } + } + + foreach (var fb in FinallyBlocks) + { + if (Log is not null) Log.CurrentTestBlock = fb.Type.ToString(); + Log?.Debug($"Starting finally block: {fb.Type}"); + + if (!TryGetBlock(testCaseScope, fb, out var finallyBlockInstance)) continue; + if (!TrySetBlockProperties(testCaseScope, fb, finallyBlockInstance)) continue; + if (!TryGetExecuteArguments(testCaseScope, fb, out List executeArgs)) continue; + + if (!TryRunBlock(fb, finallyBlockInstance, executeArgs)) + { + Log?.Critical($"Finally block failed: {FinallyBlockExceptions.LastOrDefault()}"); + } + } + + if (TestBlockException is null) + { + Passed = true; + Log?.Info("Test case finished successfully."); + } + else + { + Log?.Critical($"Test case failed: {TestBlockException}"); + } + } + + services.Dispose(); + + // This seems... gross. Revisit after sleeping. + // Maybe call out in a code review. + if (TestBlockException is not null + && (!ThrowOnFinallyBlockException + || !FinallyBlockExceptions.Any())) + { + throw new TestCaseException("Test case failed.", TestBlockException); + } + else if(TestBlockException is not null + && ThrowOnFinallyBlockException + && FinallyBlockExceptions.Any()) + { + FinallyBlockExceptions.Insert(0, TestBlockException); + throw new AggregateException("Test case failed and finally blocks failed.", + FinallyBlockExceptions); + } + else if(TestBlockException is null + && ThrowOnFinallyBlockException + && FinallyBlockExceptions.Any()) + { + throw new AggregateException("Test case succeeded, but one or more finally blocks failed.", + FinallyBlockExceptions); + } + + //if (tce is not null) throw tce; + } + + // Does it make sense for testBlock to be nullable? + // On one hand, a return of 'true' implies it will never be null. + // On the other hand, if we modify this code and accidentaly remove/forgot a 'false' check, + // it would be nice to be forced to null check. + // Might be worth setting testBlock to be non-nullable and use temp vars as the nullable type? + private bool TryGetBlock(IServiceScope scope, Block block, out object blockInstance) + { + HandleFinallyBlock( + block, + () => Log?.Debug($"Attempting to activate test block: {block.Type}"), + () => Log?.Debug($"Attempting to activate finally block: {block.Type}") + ); + bool result = false; + object? foundBlock = null; + try + { + foundBlock = scope.ServiceProvider.GetService(block.Type); + // What happens in the below scenario? + //blockInstance = scope.ServiceProvider.GetService(block.Type); + if (foundBlock is null) + { + HandleFinallyBlock( + block, + () => TestBlockException = new InvalidOperationException($"Unable to find test block: {block.Type}"), + () => FinallyBlockExceptions.Add(new InvalidOperationException($"Unable to find finally block: {block.Type}")) + ); + } + } + catch (InvalidOperationException e) + { + // Only try to re-build the test block if we get an InvalidOperationException. + // That implies the block was found but could not be activated. + // Also... can this message be clearer? Not sure what will make sense to people. + Log?.Debug($"Unable to activate from DI service, attempting to re-build block: {block.Type}. Original error: {e}"); + + _ = TryBuildBlock(scope, block, out foundBlock); + } + + if(foundBlock is not null) + { + blockInstance = foundBlock; + result = true; + } + else + { + // Is this the best way to do this? + // Or should blockInstance be nullable? + blockInstance = new object(); + } + + return result; + } + + // Notes for documentation: + // ... First level dependencies should be validated at build time. + // ... Second level dependencies are not and can fail. + // ... This mainly affects objects returned by test blocks. + // ... Best practice is to add as much as possible via AddDependency methods and *only* return items from test blocks that *have* to be. + // ... E.G. this is fine: + // ... ... TestBlock1 - returns bool + // ... ... TestBlock2 - needs bool + // ... This starts to get problematic and requires extra attention to ensure it's absolutely necesssary: + // ... ... TestBlock1 - returns bool + // ... ... TestBlock2 - needs ObjectA which needs bool + private bool TryBuildBlock(IServiceScope scope, Block block, out object? blockInstance) + { + List blockParams = new(); + foreach (ParameterInfo? c in block.ConstructorParams) + { + object? obj = ActivateObject(scope, block, c.ParameterType, "constructor argument"); + if (obj is null) + { + if(!CheckForITestLogger(c.ParameterType)) + { + blockInstance = null; + return false; + } + } + + blockParams.Add(obj); + } + blockInstance = Activator.CreateInstance(block.Type, + BindingFlags.CreateInstance | + BindingFlags.Public | + BindingFlags.Instance | + BindingFlags.OptionalParamBinding, + null, + blockParams.ToArray(), + CultureInfo.CurrentCulture); + + return true; + } + + private bool TrySetBlockProperties(IServiceScope scope, Block block, object blockInstance) + { + foreach (PropertyInfo? prop in block.PropertyParams) + { + if (!prop.CanWrite) + { + Log?.Debug($"Skipping property {prop}. No setter found."); + continue; + } + + object? obj = ActivateObject(scope, block, prop.PropertyType, "property"); + if (obj is null) + { + if(CheckForITestLogger(prop.PropertyType)) + { + continue; + } + + return false; + } + + prop.SetValue(blockInstance, obj); + Log?.TestBlockInput(obj); + } + + return true; + } + + private bool TryGetExecuteArguments(IServiceScope scope, Block block, out List executeArgs) + { + executeArgs = new List(); + foreach (ParameterInfo? ep in block.ExecuteParams) + { + object? obj = null; + if(block.ExecuteArgumentOverrides.Count > 0) + { + block.ExecuteArgumentOverrides.TryGetValue(ep.ParameterType, out obj); + } + + if(obj is null) + { + obj = ActivateObject(scope, block, ep.ParameterType, "execute method argument"); + if (obj is null) + { + if (CheckForITestLogger(ep.ParameterType)) + { + executeArgs.Add(null); + continue; + } + + return false; + } + } + + executeArgs.Add(obj); + Log?.TestBlockInput(obj); + } + + return true; + } + + private object? ActivateObject(IServiceScope scope, Block block, Type objectType, string targetMember) // Probably need to come up with a better name than 'targetMember'. + { + if (!BlockOutput.TryGetValue(objectType, out object? obj)) + { + try + { + obj = scope.ServiceProvider.GetService(objectType); + // Is the below check worth it? + // It is avoided if the test block asks for an interface if the dependency is implementing an interface. + // HOWEVER, this would facilitate injecting multiple different implementations in a test. + if(obj is null) + { + foreach(var i in objectType.GetInterfaces()) + { + IEnumerable objs = scope.ServiceProvider.GetServices(i); + obj = objs.FirstOrDefault(o => o?.GetType() == objectType); + if (obj is not null) break; + } + } + } + catch (InvalidOperationException e) + { + HandleFinallyBlock( + block, + () => TestBlockException = new InvalidOperationException( + $"Test Block - {block.Type} - Error attempting to activate {targetMember}: {objectType}: {e}"), + () => FinallyBlockExceptions.Add(new InvalidOperationException( + $"Finally Block = {block.Type} - Error attempting to activate {targetMember}: {objectType}: {e}")) + ); + } + } + + // If we've already set an exception, i.e. the GetService call above failed, don't override it. + // This is to account for two different scenarios: a dependency is not present (below) vs. a dependency is present but failed to activate (above). + if (obj is null && TestBlockException is null) + { + HandleFinallyBlock( + block, + () => TestBlockException = new InvalidOperationException( + $"Test Block - {block.Type} - Unable to find {targetMember}: {objectType}"), + () => FinallyBlockExceptions.Add(new InvalidOperationException( + $"Finally Block = {block.Type} - Unable to find {targetMember}: {objectType}")) + ); + } + + return obj; + } + + private bool TryRunBlock(Block block, object blockInstance, List executeArgs) + { + bool result = false; + + HandleFinallyBlock( + block, + () => Log?.Debug($"Executing test block: {block.Type}"), + () => Log?.Debug($"Executing finally block: {block.Type}") + ); + + try + { + object? output = block.ExecuteMethod.Invoke(blockInstance, executeArgs.ToArray()); + if (output is not null) + { + Log?.TestBlockOutput(output); + BlockOutput.Remove(output.GetType()); + BlockOutput.Add(output.GetType(), output); + } + result = true; + } + catch (TargetInvocationException ex) + { + HandleFinallyBlock( + block, + () => TestBlockException = ex.InnerException, + () => FinallyBlockExceptions.Add(ex.InnerException) + ); + } + catch (ArgumentException ex) + { + HandleFinallyBlock( + block, + () => TestBlockException = ex, + () => FinallyBlockExceptions.Add(ex) + ); + } + catch (TargetParameterCountException ex) + { + ex.Data.Add("AdditionalInfo", "Test block failed: Mismatched count between Execute method arguments and supplied dependencies."); + HandleFinallyBlock( + block, + () => TestBlockException = ex, + () => FinallyBlockExceptions.Add(ex) + ); + } + + if (result) Log?.Debug($"Test block completed successfully."); + return result; + } + + private static void HandleFinallyBlock(Block block, Action testBlockAction, Action finallyBlockAction) + { + if (block.IsFinallyBlock) + { + finallyBlockAction(); + } + else + { + testBlockAction(); + } + } + + // Type checking every single time we find no dependency seems a bit inefficient. + // Call this out specifically in a code review. + private bool CheckForITestLogger(Type type) + { + bool isLogger = false; + if (!HasLogger && type == typeof(ITestCaseLogger)) + { + TestBlockException = null; + if (FinallyBlockExceptions.Count > 0) + { + FinallyBlockExceptions.Remove(FinallyBlockExceptions.Last()); + } + isLogger = true; + } + return isLogger; + } + } +} diff --git a/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework/TestCaseException.cs b/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework/TestCaseException.cs index 8c1a0d74..59e6d1fd 100644 --- a/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework/TestCaseException.cs +++ b/IntelliTect.TestTools.TestFramework/IntelliTect.TestTools.TestFramework/TestCaseException.cs @@ -23,12 +23,12 @@ public TestCaseException(SerializationInfo info, StreamingContext context) : bas ResourceReferenceProperty = info.GetString("ResourceReferenceProperty"); } - public string ResourceReferenceProperty { get; set; } + public string ResourceReferenceProperty { get; set; } = ""; [SecurityPermission(SecurityAction.Demand, SerializationFormatter = true)] public override void GetObjectData(SerializationInfo info, StreamingContext context) { - if (info == null) throw new ArgumentNullException(nameof(info)); + if (info is null) throw new ArgumentNullException(nameof(info)); info.AddValue("ResourceReferenceProperty", ResourceReferenceProperty); base.GetObjectData(info, context); } diff --git a/testframework-pipeline.yml b/testframework-pipeline.yml index fc606aca..7d546266 100644 --- a/testframework-pipeline.yml +++ b/testframework-pipeline.yml @@ -14,7 +14,7 @@ variables: solution: '**/IntelliTect.TestTools.TestFramework.slnf' buildPlatform: 'Any CPU' buildConfiguration: 'Release' - version: '1.2.1' + version: '2.0.0-beta' steps: - task: NuGetToolInstaller@1