From 9b4650ba37a31b30f9e6c39d6af25ce2efaceefa Mon Sep 17 00:00:00 2001 From: Ian Cooper Date: Mon, 6 Nov 2023 08:29:12 +0000 Subject: [PATCH] Rebase from V9 Binary payloads (#2560) * First pass with Sqlite; errors * First pass at binary payload with Sqlite * Fix issues with SQL Parameter naming * Problem with Sqlite * Try to get Sqlite tests running * Erros with Sqlite * Fix Sqlite Inbox * Support binary payloads in Sqlite * Fix MySQL to support binary; Remove Sync suffix from Outbox name * Ensure bytes are written raw, not converted to a string, which is lossy * Run SQL Server tests on GA * Binary support for MSSQL; some naming clean up for Outboxes. Ensure that we encode * remove obsolete sql data types * Fix missing partition key header * Sqlite should support partition key * sqlite supports writing kafka partition key * Move Postgres to new format; add binary and partitionkey support * ensure consistency of headers * Fix unix timestamps * Fix issue with order of paged read * Fix tests broken now that headers don't auto-round times. * Fix up sql * Reduce precision of timestamp tests to improve success rate * Add an example for Kafka binary to make my life easier * Clean up errors in SQL tests * fix up sqlite * Fix mysql messages * missing files * Use the CreatedTime to ensure that we get the correct time * Fix SQL lite order by timestamp * Missing direction on sort * Make sure we can build * Kafka and Sqlite working with Serdes and Binary * Use RelationalDatebaseConfiguration not derived classes * Alter the interfaces to outboxes, to prevent need to cast * Fix base type * Move to new model for Dbconnection provision * Breadth of interface for async scenarios * Prevent sweeper running multiple threads * unit of work and connection provider * Ensure we register the new interfaces * entity framework is a type of transaction provider * Add commit and rollback to the provider * Binary not being set; relational db connection not transaction * use transactionconnectionprovider for lifetime * update non-binary Dapper sample * Flow move back to box transaction provider out * Move to a single abstraction for boxtransactionprovider * Modify for generic AddBrighter - need shifting of generic parameters to Outbox * Tests don't apply any more, default to an in memory outbox * Fix failing test * Fix failing mssql tests that open the connection * Make archiver non-generic * Make derived outboxes call useexternaloutbox * safety dance. * Builds; tests and samples need debugging * Ensure that we use the transaction on an external bus with an outbox * use reflection to remove need to pass generic type parameters * Remove DynamoDb extension for creation of outbox * Fix how we get the message mapper and transformer * Sample fixes * Was not marking dispatched as missing background thread * Add missing migrations to support multiple production databases * Dapper support for all dbs * safety dance * remove post overloads * safety dance * Fix issues with PostgresSql generated via Dapper.Extensions and schema created by FluentMigrator * Safety Dance prior to debug changes * Drop Dapper Extensions, simpler debugging * Move to Dapper; fix database access * Fixes to salutations for supporting all databases * Fix naming issues for the Salutations project * safewt dance * Fix DDL for Postgres * Fix exists SQL for inbox * Remove old DDL statements; improved exists checks * Improved outbox creation handling * Fix PostgreSQL Inbox issues * Fix inbox and outbox creation * Allow different transports * Ensure Kafka works on sample * Merged Kafka project, no longer needed * Remove redundant example elements, and improve README.md * Migrate ef core examples;fix issues with target framework type * Get SalutationAnalytics.csproj working with hostbuilder changes * Fix DynamoDb sample and tools * Remove out of date readmes; switch Dapper default to RMQ * ephemera that probably need removing from source control --- .../SqliteOutbox.cs | 4 ++++ ...hter.PostgreSql.EntityFrameworkCore.csproj | 1 - .../Paramore.Brighter.PostgreSql.csproj | 1 - .../SqliteTestHelper.cs | 22 +++++++++---------- 4 files changed, 15 insertions(+), 13 deletions(-) diff --git a/src/Paramore.Brighter.Outbox.Sqlite/SqliteOutbox.cs b/src/Paramore.Brighter.Outbox.Sqlite/SqliteOutbox.cs index 79f1846635..f74b80b0fe 100644 --- a/src/Paramore.Brighter.Outbox.Sqlite/SqliteOutbox.cs +++ b/src/Paramore.Brighter.Outbox.Sqlite/SqliteOutbox.cs @@ -199,7 +199,11 @@ protected override async Task ReadFromStoreAsync( } finally { +#if NETSTANDARD2_0 + connection.Close(); +#else await connection.CloseAsync(); +#endif } } } diff --git a/src/Paramore.Brighter.PostgreSql.EntityFrameworkCore/Paramore.Brighter.PostgreSql.EntityFrameworkCore.csproj b/src/Paramore.Brighter.PostgreSql.EntityFrameworkCore/Paramore.Brighter.PostgreSql.EntityFrameworkCore.csproj index dc17537644..54f1228a35 100644 --- a/src/Paramore.Brighter.PostgreSql.EntityFrameworkCore/Paramore.Brighter.PostgreSql.EntityFrameworkCore.csproj +++ b/src/Paramore.Brighter.PostgreSql.EntityFrameworkCore/Paramore.Brighter.PostgreSql.EntityFrameworkCore.csproj @@ -46,5 +46,4 @@ - diff --git a/src/Paramore.Brighter.PostgreSql/Paramore.Brighter.PostgreSql.csproj b/src/Paramore.Brighter.PostgreSql/Paramore.Brighter.PostgreSql.csproj index 3a52572399..8e5a0741a8 100644 --- a/src/Paramore.Brighter.PostgreSql/Paramore.Brighter.PostgreSql.csproj +++ b/src/Paramore.Brighter.PostgreSql/Paramore.Brighter.PostgreSql.csproj @@ -25,5 +25,4 @@ - diff --git a/tests/Paramore.Brighter.Sqlite.Tests/SqliteTestHelper.cs b/tests/Paramore.Brighter.Sqlite.Tests/SqliteTestHelper.cs index 402112b6df..1a5e1a781a 100644 --- a/tests/Paramore.Brighter.Sqlite.Tests/SqliteTestHelper.cs +++ b/tests/Paramore.Brighter.Sqlite.Tests/SqliteTestHelper.cs @@ -28,25 +28,25 @@ public SqliteTestHelper(bool binaryMessagePayload = false) public void SetupCommandDb() { - connectionStringPath = GetUniqueTestDbPathAndCreateDir(); - ConnectionString = $"DataSource=\"{connectionStringPath}\""; - CreateDatabaseWithTable(ConnectionString, SqliteInboxBuilder.GetDDL(TableName)); + _connectionStringPath = GetUniqueTestDbPathAndCreateDir(); + ConnectionString = $"DataSource=\"{_connectionStringPath}\""; + CreateDatabaseWithTable(ConnectionString, SqliteInboxBuilder.GetDDL(InboxTableName)); } public void SetupMessageDb() { - connectionStringPath = GetUniqueTestDbPathAndCreateDir(); - ConnectionString = $"DataSource=\"{connectionStringPath}\""; - CreateDatabaseWithTable(ConnectionString, SqliteOutboxBuilder.GetDDL(TableName_Messages)); + _connectionStringPath = GetUniqueTestDbPathAndCreateDir(); + ConnectionString = $"DataSource=\"{_connectionStringPath}\""; + CreateDatabaseWithTable(ConnectionString, SqliteOutboxBuilder.GetDDL(OutboxTableName, hasBinaryMessagePayload: _binaryMessagePayload)); } private string GetUniqueTestDbPathAndCreateDir() { var testRootPath = Directory.GetCurrentDirectory(); var guidInPath = Guid.NewGuid().ToString(); - connectionStringPathDir = Path.Combine(Path.Combine(Path.Combine(testRootPath, "bin"), "TestResults"), guidInPath); - Directory.CreateDirectory(connectionStringPathDir); - return Path.Combine(connectionStringPathDir, $"test{guidInPath}.db"); + _connectionStringPathDir = Path.Combine(Path.Combine(Path.Combine(testRootPath, "bin"), "TestResults"), guidInPath); + Directory.CreateDirectory(_connectionStringPathDir); + return Path.Combine(_connectionStringPathDir, $"test{guidInPath}.db"); } public async Task CleanUpDbAsync() @@ -55,8 +55,8 @@ public async Task CleanUpDbAsync() { //add 1 MS delay to allow the file to be released await Task.Delay(1); - File.Delete(connectionStringPath); - Directory.Delete(connectionStringPathDir, true); + File.Delete(_connectionStringPath); + Directory.Delete(_connectionStringPathDir, true); } catch (Exception e) {