From 00e69189bb7ea1f7416a99b95f5f103fdd2612bc Mon Sep 17 00:00:00 2001 From: Arne Dumarey Date: Wed, 8 Jan 2025 14:30:15 +0100 Subject: [PATCH] refactor(producer-snapshot): converted to console app --- .../BuildingSnapshotProducer.cs | 40 +++ .../{ApiModule.cs => ProducerModule.cs} | 4 +- .../Infrastructure/Program.cs | 175 +++++++++-- .../Infrastructure/Startup.cs | 275 ++++++++---------- .../Projections/ProjectionsController.cs | 24 -- .../appsettings.json | 2 +- 6 files changed, 312 insertions(+), 208 deletions(-) create mode 100644 src/BuildingRegistry.Producer.Snapshot.Oslo/BuildingSnapshotProducer.cs rename src/BuildingRegistry.Producer.Snapshot.Oslo/Infrastructure/Modules/{ApiModule.cs => ProducerModule.cs} (99%) delete mode 100644 src/BuildingRegistry.Producer.Snapshot.Oslo/Projections/ProjectionsController.cs diff --git a/src/BuildingRegistry.Producer.Snapshot.Oslo/BuildingSnapshotProducer.cs b/src/BuildingRegistry.Producer.Snapshot.Oslo/BuildingSnapshotProducer.cs new file mode 100644 index 000000000..126668942 --- /dev/null +++ b/src/BuildingRegistry.Producer.Snapshot.Oslo/BuildingSnapshotProducer.cs @@ -0,0 +1,40 @@ +namespace BuildingRegistry.Producer.Snapshot.Oslo +{ + using System; + using System.Threading; + using System.Threading.Tasks; + using Be.Vlaanderen.Basisregisters.Projector.ConnectedProjections; + using Microsoft.Extensions.Hosting; + using Microsoft.Extensions.Logging; + + public sealed class SnapshotProducers : BackgroundService + { + private readonly IConnectedProjectionsManager _projectionsManager; + private readonly IHostApplicationLifetime _hostApplicationLifetime; + private readonly ILogger _logger; + + public SnapshotProducers( + IConnectedProjectionsManager projectionsManager, + IHostApplicationLifetime hostApplicationLifetime, + ILoggerFactory loggerFactory) + { + _projectionsManager = projectionsManager; + _hostApplicationLifetime = hostApplicationLifetime; + _logger = loggerFactory.CreateLogger(); + } + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + try + { + await _projectionsManager.Start(stoppingToken); + } + catch (Exception exception) + { + _logger.LogCritical(exception, $"An error occurred while starting the {nameof(SnapshotProducers)}."); + _hostApplicationLifetime.StopApplication(); + throw; + } + } + } +} diff --git a/src/BuildingRegistry.Producer.Snapshot.Oslo/Infrastructure/Modules/ApiModule.cs b/src/BuildingRegistry.Producer.Snapshot.Oslo/Infrastructure/Modules/ProducerModule.cs similarity index 99% rename from src/BuildingRegistry.Producer.Snapshot.Oslo/Infrastructure/Modules/ApiModule.cs rename to src/BuildingRegistry.Producer.Snapshot.Oslo/Infrastructure/Modules/ProducerModule.cs index 1f505a3ca..e1120c4f5 100644 --- a/src/BuildingRegistry.Producer.Snapshot.Oslo/Infrastructure/Modules/ApiModule.cs +++ b/src/BuildingRegistry.Producer.Snapshot.Oslo/Infrastructure/Modules/ProducerModule.cs @@ -22,13 +22,13 @@ namespace BuildingRegistry.Producer.Snapshot.Oslo.Infrastructure.Modules using Microsoft.Extensions.Logging; using NodaTime; - public class ApiModule : Module + public class ProducerModule : Module { private readonly IConfiguration _configuration; private readonly IServiceCollection _services; private readonly ILoggerFactory _loggerFactory; - public ApiModule( + public ProducerModule( IConfiguration configuration, IServiceCollection services, ILoggerFactory loggerFactory) diff --git a/src/BuildingRegistry.Producer.Snapshot.Oslo/Infrastructure/Program.cs b/src/BuildingRegistry.Producer.Snapshot.Oslo/Infrastructure/Program.cs index 2f4bfcfe7..39daf702d 100644 --- a/src/BuildingRegistry.Producer.Snapshot.Oslo/Infrastructure/Program.cs +++ b/src/BuildingRegistry.Producer.Snapshot.Oslo/Infrastructure/Program.cs @@ -1,39 +1,166 @@ -namespace BuildingRegistry.Producer.Snapshot.Oslo.Infrastructure +namespace BuildingRegistry.Producer.Snapshot.Oslo.Infrastructure { - using Be.Vlaanderen.Basisregisters.Api; + using System; + using System.IO; + using System.Linq; + using System.Threading.Tasks; + using Autofac; + using Autofac.Extensions.DependencyInjection; using Be.Vlaanderen.Basisregisters.Aws.DistributedMutex; + using Destructurama; using Microsoft.AspNetCore.Hosting; + using Microsoft.Extensions.Configuration; + using Microsoft.Extensions.DependencyInjection; + using Microsoft.Extensions.Hosting; + using Microsoft.Extensions.Logging; + using Modules; + using Serilog; + using Serilog.Debugging; + using Serilog.Extensions.Logging; - public sealed class Program + public class Program { - private Program() + protected Program() { } - public static void Main(string[] args) - => Run(new ProgramOptions + public static async Task Main(string[] args) + { + AppDomain.CurrentDomain.FirstChanceException += (_, eventArgs) => + Log.Debug( + eventArgs.Exception, + "FirstChanceException event raised in {AppDomain}.", + AppDomain.CurrentDomain.FriendlyName); + + AppDomain.CurrentDomain.UnhandledException += (_, eventArgs) => + Log.Fatal((Exception)eventArgs.ExceptionObject, "Encountered a fatal exception, exiting program."); + + Log.Information("Initializing BuildingRegistry.Producer.Snapshot.Oslo"); + + var host = new HostBuilder() + .ConfigureAppConfiguration((_, configurationBuilder) => { - Hosting = - { - HttpPort = 6016 - }, - Logging = + configurationBuilder + .SetBasePath(Directory.GetCurrentDirectory()) + .AddJsonFile("appsettings.json", optional: false, reloadOnChange: false) + .AddJsonFile($"appsettings.{Environment.MachineName.ToLowerInvariant()}.json", optional: true, reloadOnChange: false) + .AddEnvironmentVariables() + .AddCommandLine(args); + }) + .ConfigureLogging((hostContext, loggingBuilder) => + { + SelfLog.Enable(Console.WriteLine); + Log.Logger = new LoggerConfiguration() //NOSONAR logging configuration is safe + .ReadFrom.Configuration(hostContext.Configuration) + .Enrich.FromLogContext() + .Enrich.WithMachineName() + .Enrich.WithThreadId() + .Enrich.WithEnvironmentUserName() + .Destructure.JsonNetTypes() + .CreateLogger(); + loggingBuilder.ClearProviders(); + loggingBuilder.AddSerilog(Log.Logger); + }) + .ConfigureServices((hostContext, services) => + { + var healthChecksBuilder = services.AddHealthChecks(); + var connectionStrings = hostContext.Configuration + .GetSection("ConnectionStrings") + .GetChildren(); + + foreach (var connectionString in connectionStrings + .Where(x => !x.Value.Contains("host", StringComparison.OrdinalIgnoreCase))) { - WriteTextToConsole = false, - WriteJsonToConsole = false - }, - Runtime = + healthChecksBuilder.AddSqlServer( + connectionString.Value, + name: $"sqlserver-{connectionString.Key.ToLowerInvariant()}", + tags: new[] { "db", "sql", "sqlserver" }); + } + + foreach (var connectionString in connectionStrings + .Where(x => x.Value.Contains("host", StringComparison.OrdinalIgnoreCase))) { - CommandLineArgs = args - }, - MiddlewareHooks = + healthChecksBuilder.AddNpgSql( + connectionString.Value, + name: $"npgsql-{connectionString.Key.ToLowerInvariant()}", + tags: new[] { "db", "sql", "npgsql" }); + } + + healthChecksBuilder.AddDbContextCheck( + $"dbcontext-{nameof(ProducerContext).ToLowerInvariant()}", + tags: new[] { "db", "sql", "sqlserver" }); + + var origins = hostContext.Configuration + .GetSection("Cors") + .GetChildren() + .Select(c => c.Value) + .ToArray(); + + foreach (var origin in origins) { - ConfigureDistributedLock = DistributedLockOptions.LoadFromConfiguration + services.AddCors(options => + { + options.AddDefaultPolicy(builder => + { + builder.WithOrigins(origin); + }); + }); } - }); + }) + .UseServiceProviderFactory(new AutofacServiceProviderFactory()) + .ConfigureContainer((hostContext, builder) => + { + var services = new ServiceCollection(); + var loggerFactory = new SerilogLoggerFactory(Log.Logger); + + builder.RegisterModule(new ProducerModule(hostContext.Configuration, services, loggerFactory)); + + builder + .RegisterType() + .As() + .SingleInstance(); + + builder.Populate(services); + }) + .ConfigureWebHostDefaults(webHostBuilder => + webHostBuilder + .UseStartup() + .UseKestrel()) + .UseConsoleLifetime() + .Build(); + + Log.Information("Starting BuildingRegistry.Producer.Snapshot.Oslo"); + + var logger = host.Services.GetRequiredService>(); + var configuration = host.Services.GetRequiredService(); + + try + { + await DistributedLock.RunAsync( + async () => { await host.RunAsync().ConfigureAwait(false); }, + DistributedLockOptions.LoadFromConfiguration(configuration), + logger) + .ConfigureAwait(false); + } + catch (AggregateException aggregateException) + { + foreach (var innerException in aggregateException.InnerExceptions) + { + logger.LogCritical(innerException, "Encountered a fatal exception, exiting program."); + } + } + catch (Exception e) + { + logger.LogCritical(e, "Encountered a fatal exception, exiting program."); + Log.CloseAndFlush(); - private static void Run(ProgramOptions options) - => new WebHostBuilder() - .UseDefaultForApi(options) - .RunWithLock(); + // Allow some time for flushing before shutdown. + await Task.Delay(500, default); + throw; + } + finally + { + logger.LogInformation("Stopping..."); + } + } } } diff --git a/src/BuildingRegistry.Producer.Snapshot.Oslo/Infrastructure/Startup.cs b/src/BuildingRegistry.Producer.Snapshot.Oslo/Infrastructure/Startup.cs index bf200ca29..5f92c78fc 100644 --- a/src/BuildingRegistry.Producer.Snapshot.Oslo/Infrastructure/Startup.cs +++ b/src/BuildingRegistry.Producer.Snapshot.Oslo/Infrastructure/Startup.cs @@ -1,181 +1,142 @@ -namespace BuildingRegistry.Producer.Snapshot.Oslo.Infrastructure +namespace BuildingRegistry.Producer.Snapshot.Oslo.Infrastructure { using System; + using System.Collections.Generic; using System.Linq; - using System.Reflection; - using Asp.Versioning.ApiExplorer; - using Autofac; - using Autofac.Extensions.DependencyInjection; - using Be.Vlaanderen.Basisregisters.Api; - using Be.Vlaanderen.Basisregisters.Projector; - using BuildingRegistry.Infrastructure.Modules; - using Configuration; + using System.Net.Mime; + using System.Threading; + using System.Threading.Tasks; + using Be.Vlaanderen.Basisregisters.AspNetCore.Mvc.Formatters.Json; + using Be.Vlaanderen.Basisregisters.Projector.ConnectedProjections; + using Be.Vlaanderen.Basisregisters.Projector.Controllers; using Microsoft.AspNetCore.Builder; - using Microsoft.AspNetCore.Hosting; + using Microsoft.AspNetCore.Http; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; - using Microsoft.Extensions.Diagnostics.HealthChecks; - using Microsoft.Extensions.Hosting; - using Microsoft.Extensions.Logging; - using Microsoft.OpenApi.Models; - using Modules; - - /// Represents the startup process for the application. - public class Startup + using Newtonsoft.Json; + using SqlStreamStore; + + /// + /// Extract to Library when working correctly + /// + public static class ApplicationBuilderProjectorExtensions { - private const string DatabaseTag = "db"; + public static IApplicationBuilder UseProjectorEndpoints( + this IApplicationBuilder builder, + string baseUrl, + JsonSerializerSettings? jsonSerializerSettings) + { + ArgumentNullException.ThrowIfNull(baseUrl); + + builder.UseEndpoints(endpoints => + { + endpoints.MapGet("/v1/projections", async context => { await GetProjections(builder, context, baseUrl, jsonSerializerSettings); }); + endpoints.MapGet("/projections", async context => { await GetProjections(builder, context, baseUrl, jsonSerializerSettings); }); + + endpoints.MapPost("/projections/start/all", async context => { await StartAll(builder, context); }); + endpoints.MapPost("/v1/projections/start/all", async context => { await StartAll(builder, context); }); - private IContainer _applicationContainer; + endpoints.MapPost("/projections/start/{projectionId}", async context + => await StartProjection(builder, context.Request.RouteValues["projectionId"].ToString(), context)); + endpoints.MapPost("/v1/projections/start/{projectionId}", async context + => await StartProjection(builder, context.Request.RouteValues["projectionId"].ToString(), context)); - private readonly IConfiguration _configuration; - private readonly ILoggerFactory _loggerFactory; + endpoints.MapPost("/projections/stop/all", async context => { await StopAll(builder, context); }); + endpoints.MapPost("/v1/projections/stop/all", async context => { await StopAll(builder, context); }); - public Startup( - IConfiguration configuration, - ILoggerFactory loggerFactory) + endpoints.MapPost("/projections/stop/{projectionId}", async context + => await StopProjection(builder, context.Request.RouteValues["projectionId"].ToString(), context)); + endpoints.MapPost("/v1/projections/stop/{projectionId}", async context + => await StopProjection(builder, context.Request.RouteValues["projectionId"].ToString(), context)); + }); + + return builder; + } + + private static async Task StopProjection(IApplicationBuilder app, string? projectionId, HttpContext context) { - _configuration = configuration; - _loggerFactory = loggerFactory; + var manager = app.ApplicationServices.GetRequiredService(); + await manager.Stop(projectionId, CancellationToken.None); + + context.Response.StatusCode = StatusCodes.Status202Accepted; } - /// Configures services for the application. - /// The collection of services to configure the application with. - public IServiceProvider ConfigureServices(IServiceCollection services) + private static async Task StopAll(IApplicationBuilder app, HttpContext context) { - var baseUrl = _configuration.GetValue("BaseUrl"); - var baseUrlForExceptions = baseUrl.EndsWith("/") - ? baseUrl.Substring(0, baseUrl.Length - 1) - : baseUrl; - - services - .ConfigureDefaultForApi( - new StartupConfigureOptions - { - Cors = - { - Origins = _configuration - .GetSection("Cors") - .GetChildren() - .Select(c => c.Value) - .ToArray()! - }, - Server = - { - BaseUrl = baseUrlForExceptions - }, - Swagger = - { - ApiInfo = (provider, description) => new OpenApiInfo - { - Version = description.ApiVersion.ToString(), - Title = "Basisregisters Vlaanderen Building Registry API", - Description = GetApiLeadingText(description), - Contact = new OpenApiContact - { - Name = "Digitaal Vlaanderen", - Email = "digitaal.vlaanderen@vlaanderen.be", - Url = new Uri("https://legacy.basisregisters.vlaanderen") - } - }, - XmlCommentPaths = [typeof(Startup).GetTypeInfo().Assembly.GetName().Name!] - }, - MiddlewareHooks = - { - FluentValidation = fv => fv.RegisterValidatorsFromAssemblyContaining(), - - AfterHealthChecks = health => - { - var connectionStrings = _configuration - .GetSection("ConnectionStrings") - .GetChildren() - .ToArray(); - - foreach (var connectionString in connectionStrings - .Where(x => !x.Value!.Contains("host", StringComparison.OrdinalIgnoreCase))) - { - health.AddSqlServer( - connectionString.Value!, - name: $"sqlserver-{connectionString.Key.ToLowerInvariant()}", - tags: [ DatabaseTag, "sql", "sqlserver" ]); - } - - foreach (var connectionString in connectionStrings - .Where(x => x.Value!.Contains("host", StringComparison.OrdinalIgnoreCase))) - { - health.AddNpgSql( - connectionString.Value!, - name: $"npgsql-{connectionString.Key.ToLowerInvariant()}", - tags: [ DatabaseTag, "sql", "npgsql" ]); - } - - health.AddDbContextCheck( - $"dbcontext-{nameof(ProducerContext).ToLowerInvariant()}", - tags: new[] {DatabaseTag, "sql", "sqlserver"}); - } - } - }); - - var containerBuilder = new ContainerBuilder(); - containerBuilder.RegisterModule(new LoggingModule(_configuration, services)); - containerBuilder.RegisterModule(new ApiModule(_configuration, services, _loggerFactory)); - _applicationContainer = containerBuilder.Build(); - - return new AutofacServiceProvider(_applicationContainer); + var manager = app.ApplicationServices.GetRequiredService(); + await manager.Stop(CancellationToken.None); + + context.Response.StatusCode = StatusCodes.Status202Accepted; } - public void Configure( - IServiceProvider serviceProvider, - IApplicationBuilder app, - IWebHostEnvironment env, - IHostApplicationLifetime appLifetime, - ILoggerFactory loggerFactory, - IApiVersionDescriptionProvider apiVersionProvider, - HealthCheckService healthCheckService) + private static async Task StartProjection(IApplicationBuilder app, string? projectionId, HttpContext context) { - StartupHelpers.CheckDatabases(healthCheckService, DatabaseTag, loggerFactory).GetAwaiter().GetResult(); + var manager = app.ApplicationServices.GetRequiredService(); - app - .UseDefaultForApi(new StartupUseOptions - { - Common = - { - ApplicationContainer = _applicationContainer, - ServiceProvider = serviceProvider, - HostingEnvironment = env, - ApplicationLifetime = appLifetime, - LoggerFactory = loggerFactory - }, - Api = - { - VersionProvider = apiVersionProvider, - Info = groupName => $"Basisregisters Vlaanderen - Building Registry API {groupName}", - CSharpClientOptions = - { - ClassName = "BuildingRegistryProducer", - Namespace = "Be.Vlaanderen.Basisregisters" - }, - TypeScriptClientOptions = - { - ClassName = "BuildingRegistryProducer" - } - }, - MiddlewareHooks = - { - AfterMiddleware = x => x.UseMiddleware() - } - }) - - .UseProjectionsManager(new ProjectionsManagerOptions + await manager.Start(projectionId, CancellationToken.None); + + context.Response.StatusCode = StatusCodes.Status202Accepted; + } + + private static async Task StartAll(IApplicationBuilder app, HttpContext context) + { + var manager = app.ApplicationServices.GetRequiredService(); + await manager.Start(CancellationToken.None); + + context.Response.StatusCode = StatusCodes.Status202Accepted; + } + + private static async Task GetProjections( + IApplicationBuilder app, + HttpContext context, + string baseUrl, + JsonSerializerSettings? jsonSerializerSettings = null) + { + var manager = app.ApplicationServices.GetRequiredService(); + var streamStore = app.ApplicationServices.GetRequiredService(); + + var registeredConnectedProjections = manager + .GetRegisteredProjections() + .ToList(); + var projectionStates = await manager.GetProjectionStates(CancellationToken.None); + var responses = registeredConnectedProjections.Aggregate( + new List(), + (list, projection) => { - Common = - { - ServiceProvider = serviceProvider, - ApplicationLifetime = appLifetime - } + var projectionState = projectionStates.SingleOrDefault(x => x.Name == projection.Id); + list.Add(new ProjectionResponse( + projection, + projectionState, + baseUrl)); + return list; }); + + var streamPosition = await streamStore.ReadHeadPosition(); + + var projectionResponseList = new ProjectionResponseList(responses, baseUrl) + { + StreamPosition = streamPosition + }; + + var json = JsonConvert.SerializeObject(projectionResponseList, jsonSerializerSettings ?? new JsonSerializerSettings()); + + context.Response.Headers.ContentType = MediaTypeNames.Application.Json; + await context.Response.WriteAsync(json); } + } - private static string GetApiLeadingText(ApiVersionDescription description) - => $"Momenteel leest u de documentatie voor versie {description.ApiVersion} van de Basisregisters Vlaanderen Building Registry Producer Snapshot Oslo API{string.Format(description.IsDeprecated ? ", **deze API versie is niet meer ondersteund * *." : ".")}"; + public sealed class Startup + { + public void Configure(IApplicationBuilder app) + { + app.UseRouting(); + app.UseCors(); + + app.UseHealthChecks("/health"); + + var configuration = app.ApplicationServices.GetRequiredService(); + var baseUri = configuration.GetValue("BaseUrl").TrimEnd('/'); + app.UseProjectorEndpoints(baseUri, new JsonSerializerSettings().ConfigureDefaultForApi()); + } } } diff --git a/src/BuildingRegistry.Producer.Snapshot.Oslo/Projections/ProjectionsController.cs b/src/BuildingRegistry.Producer.Snapshot.Oslo/Projections/ProjectionsController.cs deleted file mode 100644 index 013f7e977..000000000 --- a/src/BuildingRegistry.Producer.Snapshot.Oslo/Projections/ProjectionsController.cs +++ /dev/null @@ -1,24 +0,0 @@ -namespace BuildingRegistry.Producer.Snapshot.Oslo.Projections -{ - using Asp.Versioning; - using Be.Vlaanderen.Basisregisters.Api; - using Be.Vlaanderen.Basisregisters.Projector.ConnectedProjections; - using Be.Vlaanderen.Basisregisters.Projector.Controllers; - using BuildingRegistry.Infrastructure; - using Microsoft.Extensions.Configuration; - - [ApiVersion("1.0")] - [ApiRoute("projections")] - public class ProjectionsController : DefaultProjectorController - { - public ProjectionsController( - IConnectedProjectionsManager connectedProjectionsManager, - IConfiguration configuration) - : base( - connectedProjectionsManager, - configuration.GetValue("BaseUrl")) - { - RegisterConnectionString(Schema.ProducerSnapshotOslo, configuration.GetConnectionString("ProducerSnapshotProjections")); - } - } -} diff --git a/src/BuildingRegistry.Producer.Snapshot.Oslo/appsettings.json b/src/BuildingRegistry.Producer.Snapshot.Oslo/appsettings.json index 8d35a5a51..f3271019d 100644 --- a/src/BuildingRegistry.Producer.Snapshot.Oslo/appsettings.json +++ b/src/BuildingRegistry.Producer.Snapshot.Oslo/appsettings.json @@ -9,7 +9,7 @@ "BuildingTopic": "dev.building.snapshot.oslo", "BuildingUnitTopic": "dev.buildingunit.snapshot.oslo", - "BaseUrl": "https://api.staging-basisregisters.vlaanderen/", + "BaseUrl": "http://localhost:5000/", "BuildingOsloNamespace": "https://data.vlaanderen.be/id/gebouw", "BuildingUnitOsloNamespace": "https://data.vlaanderen.be/id/gebouweenheid",