diff --git a/Dockerfile b/Dockerfile index cae57a7fe1..b9d1209987 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,7 +5,6 @@ COPY ./*.sln ./NuGet.Config ./ COPY ./src/*.props ./src/ COPY ./tests/*.props ./tests/ COPY ./build/packages/* ./build/packages/ -COPY ./docker/docker-compose.dcproj ./docker/ # Copy the main source project files COPY src/*/*.csproj ./ diff --git a/Exceptionless.sln b/Exceptionless.sln index 34cba061b0..04b99ad6db 100644 --- a/Exceptionless.sln +++ b/Exceptionless.sln @@ -11,7 +11,6 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution .github\workflows\build.yaml = .github\workflows\build.yaml CONTRIBUTING.md = CONTRIBUTING.md src\Directory.Build.props = src\Directory.Build.props - docker\docker-compose.yml = docker\docker-compose.yml Dockerfile = Dockerfile exceptionless.http = exceptionless.http global.json = global.json @@ -28,8 +27,6 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Exceptionless.Tests", "test EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Exceptionless.Job", "src\Exceptionless.Job\Exceptionless.Job.csproj", "{788BA00C-FFBE-42A9-92A3-89E24FC137B5}" EndProject -Project("{E53339B2-1760-4266-BCC7-CA923CBCF16C}") = "docker-compose", "docker\docker-compose.dcproj", "{9F933018-9E8B-4649-8C9A-D217B5E1C184}" -EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "http", "http", "{97ED03A0-8C49-4B15-8D93-C56AF4DDC30F}" ProjectSection(SolutionItems) = preProject tests\http\admin.http = tests\http\admin.http @@ -44,6 +41,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "http", "http", "{97ED03A0-8 tests\http\webhooks.http = tests\http\webhooks.http EndProjectSection EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Exceptionless.AppHost", "src\Exceptionless.AppHost\Exceptionless.AppHost.csproj", "{EB1AF004-A00D-4016-BA97-5E89177B0074}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -70,10 +69,10 @@ Global {788BA00C-FFBE-42A9-92A3-89E24FC137B5}.Debug|Any CPU.Build.0 = Debug|Any CPU {788BA00C-FFBE-42A9-92A3-89E24FC137B5}.Release|Any CPU.ActiveCfg = Release|Any CPU {788BA00C-FFBE-42A9-92A3-89E24FC137B5}.Release|Any CPU.Build.0 = Release|Any CPU - {9F933018-9E8B-4649-8C9A-D217B5E1C184}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {9F933018-9E8B-4649-8C9A-D217B5E1C184}.Debug|Any CPU.Build.0 = Debug|Any CPU - {9F933018-9E8B-4649-8C9A-D217B5E1C184}.Release|Any CPU.ActiveCfg = Release|Any CPU - {9F933018-9E8B-4649-8C9A-D217B5E1C184}.Release|Any CPU.Build.0 = Release|Any CPU + {EB1AF004-A00D-4016-BA97-5E89177B0074}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {EB1AF004-A00D-4016-BA97-5E89177B0074}.Debug|Any CPU.Build.0 = Debug|Any CPU + {EB1AF004-A00D-4016-BA97-5E89177B0074}.Release|Any CPU.ActiveCfg = Release|Any CPU + {EB1AF004-A00D-4016-BA97-5E89177B0074}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/docker/docker-compose.dcproj b/docker/docker-compose.dcproj deleted file mode 100644 index 1a5989b744..0000000000 --- a/docker/docker-compose.dcproj +++ /dev/null @@ -1,11 +0,0 @@ - - - - 2.1 - Linux - 9f933018-9e8b-4649-8c9a-d217b5e1c184 - - - - - \ No newline at end of file diff --git a/src/Exceptionless.AppHost/Exceptionless.AppHost.csproj b/src/Exceptionless.AppHost/Exceptionless.AppHost.csproj new file mode 100644 index 0000000000..74d6e01d75 --- /dev/null +++ b/src/Exceptionless.AppHost/Exceptionless.AppHost.csproj @@ -0,0 +1,27 @@ + + + + + + Exe + net9.0 + enable + enable + true + a9c2ddcc-e51d-4cd1-9782-96e1d74eec87 + + + + + + + + + + + + + + + + diff --git a/src/Exceptionless.AppHost/Extensions/ElasticsearchExtensions.cs b/src/Exceptionless.AppHost/Extensions/ElasticsearchExtensions.cs new file mode 100644 index 0000000000..423bb813ae --- /dev/null +++ b/src/Exceptionless.AppHost/Extensions/ElasticsearchExtensions.cs @@ -0,0 +1,122 @@ +using Aspire.Hosting.Lifecycle; +using Aspire.Hosting.Utils; +using HealthChecks.Elasticsearch; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Diagnostics.HealthChecks; + +namespace Aspire.Hosting; + +/// +/// Provides extension methods for adding Elasticsearch resources to the application model. +/// +public static class ElasticsearchBuilderExtensions +{ + private const int ElasticsearchPort = 9200; + private const int ElasticsearchInternalPort = 9300; + private const int KibanaPort = 5601; + + /// + /// Adds a Elasticsearch container to the application model. The default image is "docker.elastic.co/elasticsearch/elasticsearch". This version the package defaults to the 8.17.0 tag of the Elasticsearch container image + /// + /// The . + /// The name of the resource. This name will be used as the connection string name when referenced in a dependency. + /// The host port to bind the underlying container to. + /// A reference to the . + public static IResourceBuilder AddElasticsearch(this IDistributedApplicationBuilder builder, [ResourceName] string name, int? port = null) + { + ArgumentNullException.ThrowIfNull(builder); + ArgumentNullException.ThrowIfNull(name); + + var elasticsearch = new ElasticsearchResource(name); + + string? connectionString = null; + ElasticsearchOptions? options = null; + + builder.Eventing.Subscribe(elasticsearch, async (@event, ct) => + { + connectionString = await elasticsearch.ConnectionStringExpression.GetValueAsync(ct).ConfigureAwait(false); + if (connectionString is null) + { + throw new DistributedApplicationException($"ConnectionStringAvailableEvent was published for the '{elasticsearch.Name}' resource but the connection string was null."); + } + + options = new ElasticsearchOptions(); + options.UseServer(connectionString); + }); + + var healthCheckKey = $"{name}_check"; + builder.Services.AddHealthChecks() + .Add(new HealthCheckRegistration( + healthCheckKey, + sp => new ElasticsearchHealthCheck(options!), + failureStatus: default, + tags: default, + timeout: default)); + + return builder.AddResource(elasticsearch) + .WithImage(ElasticsearchContainerImageTags.Image, ElasticsearchContainerImageTags.Tag) + .WithImageRegistry(ElasticsearchContainerImageTags.ElasticsearchRegistry) + .WithHttpEndpoint(targetPort: ElasticsearchPort, port: port, name: ElasticsearchResource.PrimaryEndpointName) + .WithEndpoint(targetPort: ElasticsearchInternalPort, name: ElasticsearchResource.InternalEndpointName) + .WithEnvironment("discovery.type", "single-node") + .WithEnvironment("xpack.security.enabled", "false") + .WithEnvironment("action.destructive_requires_name", "false") + .WithEnvironment("ES_JAVA_OPTS", "-Xms1g -Xmx1g") + .WithHealthCheck(healthCheckKey) + .PublishAsConnectionString(); + } + + public static IResourceBuilder WithKibana(this IResourceBuilder builder, Action>? configureContainer = null, string? containerName = null) + { + ArgumentNullException.ThrowIfNull(builder); + + if (builder.ApplicationBuilder.Resources.OfType().SingleOrDefault() is { } existingKibanaResource) + { + var builderForExistingResource = builder.ApplicationBuilder.CreateResourceBuilder(existingKibanaResource); + configureContainer?.Invoke(builderForExistingResource); + return builder; + } + else + { + containerName ??= $"{builder.Resource.Name}-kibana"; + + builder.ApplicationBuilder.Services.TryAddLifecycleHook(); + + var resource = new KibanaResource(containerName); + var resourceBuilder = builder.ApplicationBuilder.AddResource(resource) + .WithImage(ElasticsearchContainerImageTags.KibanaImage, ElasticsearchContainerImageTags.Tag) + .WithImageRegistry(ElasticsearchContainerImageTags.KibanaRegistry) + .WithHttpEndpoint(targetPort: KibanaPort, name: containerName) + .WithEnvironment("xpack.security.enabled", "false") + .ExcludeFromManifest(); + + configureContainer?.Invoke(resourceBuilder); + + return builder; + } + } + + public static IResourceBuilder WithDataVolume(this IResourceBuilder builder, string? name = null) + { + ArgumentNullException.ThrowIfNull(builder); + + return builder.WithVolume(name ?? VolumeNameGenerator.CreateVolumeName(builder, "data"), "/usr/share/elasticsearch/data"); + } + + public static IResourceBuilder WithDataBindMount(this IResourceBuilder builder, string source) + { + ArgumentNullException.ThrowIfNull(builder); + ArgumentNullException.ThrowIfNull(source); + + return builder.WithBindMount(source, "/usr/share/elasticsearch/data"); + } +} + +internal static class ElasticsearchContainerImageTags +{ + public const string ElasticsearchRegistry = "docker.io"; + public const string Image = "exceptionless/elasticsearch"; + public const string KibanaRegistry = "docker.elastic.co"; + public const string KibanaImage = "kibana/kibana"; + public const string Tag = "8.17.0"; +} diff --git a/src/Exceptionless.AppHost/Extensions/ElasticsearchResource.cs b/src/Exceptionless.AppHost/Extensions/ElasticsearchResource.cs new file mode 100644 index 0000000000..ebdd1ea09d --- /dev/null +++ b/src/Exceptionless.AppHost/Extensions/ElasticsearchResource.cs @@ -0,0 +1,72 @@ +namespace Aspire.Hosting; + +/// +/// A resource that represents a Elasticsearch resource independent of the hosting model. +/// +public class ElasticsearchResource : ContainerResource, IResourceWithConnectionString +{ + // this endpoint is used for all API calls over HTTP. + // This includes search and aggregations, monitoring and anything else that uses a HTTP request. + // All client libraries will use this port to talk to Elasticsearch + internal const string PrimaryEndpointName = "http"; + + //this endpoint is a custom binary protocol used for communications between nodes in a cluster. + //For things like cluster updates, master elections, nodes joining/leaving, shard allocation + internal const string InternalEndpointName = "internal"; + + /// The name of the resource. + public ElasticsearchResource(string name) : base(name) + { + } + + private EndpointReference? _primaryEndpoint; + private EndpointReference? _internalEndpoint; + + /// + /// Gets the primary endpoint for the Elasticsearch. This endpoint is used for all API calls over HTTP. + /// + public EndpointReference PrimaryEndpoint => _primaryEndpoint ??= new(this, PrimaryEndpointName); + + /// + /// Gets the internal endpoint for the Elasticsearch. This endpoint used for communications between nodes in a cluster + /// + public EndpointReference InternalEndpoint => _internalEndpoint ??= new(this, InternalEndpointName); + + /// + /// Gets the connection string expression for the Elasticsearch + /// + public ReferenceExpression ConnectionString => + ReferenceExpression.Create($"http://{PrimaryEndpoint.Property(EndpointProperty.Host)}:{PrimaryEndpoint.Property(EndpointProperty.Port)}"); + + + /// + /// Gets the connection string expression for the Elasticsearch server for the manifest. + /// + public ReferenceExpression ConnectionStringExpression + { + get + { + if (this.TryGetLastAnnotation(out var connectionStringAnnotation)) + { + return connectionStringAnnotation.Resource.ConnectionStringExpression; + } + + return ConnectionString; + } + } + + /// + /// Gets the connection string for the Elasticsearch server. + /// + /// A to observe while waiting for the task to complete. + /// A connection string for the Elasticsearch server in the form "http://host:port". + public ValueTask GetConnectionStringAsync(CancellationToken cancellationToken = default) + { + if (this.TryGetLastAnnotation(out var connectionStringAnnotation)) + { + return connectionStringAnnotation.Resource.GetConnectionStringAsync(cancellationToken); + } + + return ConnectionString.GetValueAsync(cancellationToken); + } +} diff --git a/src/Exceptionless.AppHost/Extensions/KibanaConfigWriterHook.cs b/src/Exceptionless.AppHost/Extensions/KibanaConfigWriterHook.cs new file mode 100644 index 0000000000..67bae87b43 --- /dev/null +++ b/src/Exceptionless.AppHost/Extensions/KibanaConfigWriterHook.cs @@ -0,0 +1,37 @@ +using System.Text; +using Aspire.Hosting.Lifecycle; +using Microsoft.Extensions.DependencyInjection; + +namespace Aspire.Hosting; + +internal class KibanaConfigWriterHook : IDistributedApplicationLifecycleHook +{ + public async Task AfterEndpointsAllocatedAsync(DistributedApplicationModel appModel, CancellationToken cancellationToken) + { + if (appModel.Resources.OfType().SingleOrDefault() is not { } kibanaResource) + return; + + var elasticsearchInstances = appModel.Resources.OfType(); + + if (!elasticsearchInstances.Any()) + return; + + var hostsVariableBuilder = new StringBuilder(); + + foreach (var elasticsearchInstance in elasticsearchInstances) + { + if (elasticsearchInstance.PrimaryEndpoint.IsAllocated) + { + var connectionString = await elasticsearchInstance.GetConnectionStringAsync(); + if (hostsVariableBuilder.Length > 0) + hostsVariableBuilder.Append(","); + hostsVariableBuilder.Append(elasticsearchInstance.PrimaryEndpoint.Scheme).Append("://").Append(elasticsearchInstance.PrimaryEndpoint.ContainerHost).Append(":").Append(elasticsearchInstance.PrimaryEndpoint.Port); + } + } + + kibanaResource.Annotations.Add(new EnvironmentCallbackAnnotation(context => + { + context.EnvironmentVariables.Add("ELASTICSEARCH_HOSTS", hostsVariableBuilder.ToString()); + })); + } +} diff --git a/src/Exceptionless.AppHost/Extensions/KibanaResource.cs b/src/Exceptionless.AppHost/Extensions/KibanaResource.cs new file mode 100644 index 0000000000..7e5031a46b --- /dev/null +++ b/src/Exceptionless.AppHost/Extensions/KibanaResource.cs @@ -0,0 +1,9 @@ +namespace Aspire.Hosting; + +/// +/// A resource that represents a Kibana container. +/// +/// The name of the resource. +public class KibanaResource(string name) : ContainerResource(name) +{ +} diff --git a/src/Exceptionless.AppHost/Extensions/MinIoExtensions.cs b/src/Exceptionless.AppHost/Extensions/MinIoExtensions.cs new file mode 100644 index 0000000000..464f3985f8 --- /dev/null +++ b/src/Exceptionless.AppHost/Extensions/MinIoExtensions.cs @@ -0,0 +1,138 @@ +using Foundatio.Storage; + +namespace Aspire.Hosting; + +public static class MinIoExtensions +{ + public static IResourceBuilder AddMinIo( + this IDistributedApplicationBuilder builder, + string name, + Action? configure = null) + { + var options = new MinIoBuilder(); + configure?.Invoke(options); + + var resource = new MinIoResource(name, options.AccessKey, options.SecretKey, options.Bucket ?? "storage"); + + string? connectionString = null; + + builder.Eventing.Subscribe(resource, async (@event, ct) => + { + connectionString = await resource.ConnectionStringExpression.GetValueAsync(ct).ConfigureAwait(false); + + if (connectionString == null) + throw new DistributedApplicationException($"ResourceReadyEvent was published for the '{resource.Name}' resource but the connection string was null."); + + var storage = new S3FileStorage(o => o.ConnectionString(connectionString)); + try + { + storage.Client.PutBucketAsync(options.Bucket ?? "storage", ct).GetAwaiter().GetResult(); + } + catch + { + // ignored + } + }); + + return builder.AddResource(resource) + .WithImage(MinIoContainerImageTags.Image) + .WithImageRegistry(MinIoContainerImageTags.Registry) + .WithImageTag(MinIoContainerImageTags.Tag) + .WithArgs("server", "/data", "--console-address", $":{MinIoResource.DefaultConsolePort}") + .WithEndpoint(port: options.ApiPort, targetPort: MinIoResource.DefaultApiPort, name: MinIoResource.ApiEndpointName) + .WithHttpEndpoint(port: options.ConsolePort, targetPort: MinIoResource.DefaultConsolePort, name: MinIoResource.ConsoleEndpointName) + .ConfigureCredentials(options) + .ConfigureVolume(options); + } + + private static IResourceBuilder ConfigureCredentials( + this IResourceBuilder builder, + MinIoBuilder options) + { + return builder + .WithEnvironment("MINIO_ROOT_USER", options.AccessKey ?? "minioadmin") + .WithEnvironment("MINIO_ROOT_PASSWORD", options.SecretKey ?? "minioadmin"); + } + + private static IResourceBuilder ConfigureVolume( + this IResourceBuilder builder, + MinIoBuilder options) + { + if (!string.IsNullOrEmpty(options.DataVolumePath)) + builder = builder.WithVolume(options.DataVolumePath, "/data"); + + return builder; + } +} + +public class MinIoResource(string name, string? accessKey = null, string? secretKey = null, string? bucket = "storage") + : ContainerResource(name), IResourceWithConnectionString +{ + internal const string ApiEndpointName = "api"; + internal const string ConsoleEndpointName = "console"; + internal const int DefaultApiPort = 9000; + internal const int DefaultConsolePort = 9001; + + private EndpointReference? _apiReference; + private EndpointReference? _consoleReference; + + private EndpointReference ApiEndpoint => + _apiReference ??= new EndpointReference(this, ApiEndpointName); + + private EndpointReference ConsoleEndpoint => + _consoleReference ??= new EndpointReference(this, ConsoleEndpointName); + + public ReferenceExpression ConnectionStringExpression => + ReferenceExpression.Create( + $"ServiceUrl=http://{ApiEndpoint.Property(EndpointProperty.Host)}:{ApiEndpoint.Property(EndpointProperty.Port)};" + + $"AccessKey={AccessKey ?? "minioadmin"};" + + $"SecretKey={SecretKey ?? "minioadmin"};" + + $"Bucket={Bucket}"); + + public string? AccessKey { get; } = accessKey; + public string? SecretKey { get; } = secretKey; + public string? Bucket { get; } = bucket; +} + +public class MinIoBuilder +{ + public int? ApiPort { get; set; } + public int? ConsolePort { get; set; } + public string? AccessKey { get; set; } + public string? SecretKey { get; set; } + public string? Bucket { get; set; } + public string? DataVolumePath { get; set; } + + public MinIoBuilder WithPorts(int? apiPort = null, int? consolePort = null) + { + ApiPort = apiPort; + ConsolePort = consolePort; + return this; + } + + public MinIoBuilder WithCredentials(string accessKey, string secretKey) + { + AccessKey = accessKey; + SecretKey = secretKey; + return this; + } + + public MinIoBuilder WithBucket(string bucket) + { + Bucket = bucket; + return this; + } + + public MinIoBuilder WithDataVolume(string path) + { + DataVolumePath = path; + return this; + } +} + +internal static class MinIoContainerImageTags +{ + internal const string Registry = "docker.io"; + internal const string Image = "minio/minio"; + internal const string Tag = "RELEASE.2024-12-13T22-19-12Z"; +} diff --git a/src/Exceptionless.AppHost/Extensions/RedisExtensions.cs b/src/Exceptionless.AppHost/Extensions/RedisExtensions.cs new file mode 100644 index 0000000000..71e75f94db --- /dev/null +++ b/src/Exceptionless.AppHost/Extensions/RedisExtensions.cs @@ -0,0 +1,28 @@ +using Microsoft.Extensions.Diagnostics.HealthChecks; +using StackExchange.Redis; + +namespace Aspire.Hosting; + +public static class RedisExtensions +{ + public static IResourceBuilder WithClearCommand( + this IResourceBuilder builder) + { + builder.WithCommand( + "clear-cache", + "Clear Cache", + async _ => + { + var redisConnectionString = await builder.Resource.GetConnectionStringAsync() ?? + throw new InvalidOperationException("Unable to get the Redis connection string."); + + await using var connection = await ConnectionMultiplexer.ConnectAsync(redisConnectionString); + + await connection.GetDatabase().ExecuteAsync("FLUSHALL"); + + return CommandResults.Success(); + }, + context => context.ResourceSnapshot.HealthStatus is HealthStatus.Healthy ? ResourceCommandState.Enabled : ResourceCommandState.Disabled); + return builder; + } +} diff --git a/src/Exceptionless.AppHost/Extensions/VolumeNameGenerator.cs b/src/Exceptionless.AppHost/Extensions/VolumeNameGenerator.cs new file mode 100644 index 0000000000..6ecbc551f8 --- /dev/null +++ b/src/Exceptionless.AppHost/Extensions/VolumeNameGenerator.cs @@ -0,0 +1,65 @@ +namespace Aspire.Hosting.Utils; + +internal static class VolumeNameGenerator +{ + public static string CreateVolumeName(IResourceBuilder builder, string suffix) where T : IResource + { + if (!HasOnlyValidChars(suffix)) + { + throw new ArgumentException($"The suffix '{suffix}' contains invalid characters. Only [a-zA-Z0-9_.-] are allowed.", nameof(suffix)); + } + + // Creates a volume name with the form < c > $"{applicationName}-{sha256 of apphost path}-{resourceName}-{suffix}, e.g. "myapplication-a345f2451-postgres-data". + // Create volume name like "{Sanitize(appname).Lower()}-{sha256.Lower()}-postgres-data" + + // Compute a short hash of the content root path to differentiate between multiple AppHost projects with similar volume names + var safeApplicationName = Sanitize(builder.ApplicationBuilder.Environment.ApplicationName).ToLowerInvariant(); + var applicationHash = builder.ApplicationBuilder.Configuration["AppHost:Sha256"]![..10].ToLowerInvariant(); + var resourceName = builder.Resource.Name; + return $"{safeApplicationName}-{applicationHash}-{resourceName}-{suffix}"; + } + + public static string Sanitize(string name) + { + return string.Create(name.Length, name, static (s, name) => + { + // According to the error message from docker CLI, volume names must be of form "[a-zA-Z0-9][a-zA-Z0-9_.-]" + var nameSpan = name.AsSpan(); + + for (var i = 0; i < nameSpan.Length; i++) + { + var c = nameSpan[i]; + + s[i] = IsValidChar(i, c) ? c : '_'; + } + }); + } + + private static bool HasOnlyValidChars(string value) + { + for (var i = 0; i < value.Length; i++) + { + if (!IsValidChar(i, value[i])) + { + return false; + } + } + return true; + } + + private static bool IsValidChar(int i, char c) + { + if (i == 0 && !(char.IsAsciiLetter(c) || char.IsNumber(c))) + { + // First char must be a letter or number + return false; + } + else if (!(char.IsAsciiLetter(c) || char.IsNumber(c) || c == '_' || c == '.' || c == '-')) + { + // Subsequent chars must be a letter, number, underscore, period, or hyphen + return false; + } + + return true; + } +} diff --git a/src/Exceptionless.AppHost/Program.cs b/src/Exceptionless.AppHost/Program.cs new file mode 100644 index 0000000000..0df5bf1581 --- /dev/null +++ b/src/Exceptionless.AppHost/Program.cs @@ -0,0 +1,57 @@ +var builder = DistributedApplication.CreateBuilder(args); + +var elastic = builder.AddElasticsearch("Elasticsearch", port: 9200) + .WithLifetime(ContainerLifetime.Persistent) + .WithContainerName("Exceptionless-Elasticsearch") + .WithDataVolume("exceptionless.data.v1") + .WithKibana(b => b.WithLifetime(ContainerLifetime.Persistent).WithContainerName("Exceptionless-Kibana")); + +var storage = builder.AddMinIo("S3", s => s.WithCredentials("guest", "password").WithPorts(9000).WithBucket("ex-events")) + .WithLifetime(ContainerLifetime.Persistent) + .WithContainerName("Exceptionless-Storage"); + +var cache = builder.AddRedis("Redis", port: 6379) + .WithImageTag("7.4") + .WithLifetime(ContainerLifetime.Persistent) + .WithContainerName("Exceptionless-Redis") + .WithClearCommand() + .WithRedisInsight(b => b.WithLifetime(ContainerLifetime.Persistent).WithContainerName("Exceptionless-RedisInsight")); + +var mail = builder.AddContainer("Mail", "mailhog/mailhog") + .WithLifetime(ContainerLifetime.Persistent) + .WithContainerName("Exceptionless-Mail") + .WithEndpoint(8025, 8025, "http") + .WithEndpoint(1025, 1025); + +builder.AddProject("Jobs", "AllJobs") + .WithReference(cache) + .WithReference(elastic) + .WithReference(storage) + .WithEnvironment("ConnectionStrings:Email", "smtp://localhost:1025") + .WaitFor(elastic) + .WaitFor(cache) + .WaitFor(mail) + .WithHttpHealthCheck("/health"); + +var api = builder.AddProject("Api", "Exceptionless") + .WithReference(cache) + .WithReference(elastic) + .WithReference(storage) + .WithEnvironment("ConnectionStrings:Email", "smtp://localhost:1025") + .WithEnvironment("RunJobsInProcess", "false") + .WaitFor(elastic) + .WaitFor(cache) + .WaitFor(mail) + .WithHttpHealthCheck("/health"); + +builder.AddNpmApp("Web", "../../src/Exceptionless.Web/ClientApp", "dev") + .WithReference(api) + .WithEnvironment("ASPNETCORE_URLS", "http://localhost:5200") + .WithEndpoint(port: 5173, targetPort: 5173, scheme: "http", env: "PORT", isProxied: false); + +builder.AddNpmApp("AngularWeb", "../../src/Exceptionless.Web/ClientApp.angular", "serve") + .WithReference(api) + .WithEnvironment("ASPNETCORE_URLS", "http://localhost:5200") + .WithEndpoint(port: 5100, targetPort: 5100, scheme: "http", env: "PORT", isProxied: false); + +builder.Build().Run(); diff --git a/src/Exceptionless.AppHost/Properties/launchSettings.json b/src/Exceptionless.AppHost/Properties/launchSettings.json new file mode 100644 index 0000000000..a657132e91 --- /dev/null +++ b/src/Exceptionless.AppHost/Properties/launchSettings.json @@ -0,0 +1,42 @@ +{ + "$schema": "https://json.schemastore.org/launchsettings.json", + "profiles": { + "https": { + "commandName": "Project", + "dotnetRunMessages": true, + "launchBrowser": true, + "applicationUrl": "https://localhost:17056;http://localhost:15161", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development", + "DOTNET_ENVIRONMENT": "Development", + "DOTNET_DASHBOARD_OTLP_ENDPOINT_URL": "https://localhost:21210", + "DOTNET_RESOURCE_SERVICE_ENDPOINT_URL": "https://localhost:22299" + } + }, + "https-all": { + "commandName": "Project", + "dotnetRunMessages": true, + "launchBrowser": true, + "applicationUrl": "https://localhost:17056;http://localhost:15161", + "environmentVariables": { + "EX_ALL": "true", + "ASPNETCORE_ENVIRONMENT": "Development", + "DOTNET_ENVIRONMENT": "Development", + "DOTNET_DASHBOARD_OTLP_ENDPOINT_URL": "https://localhost:21210", + "DOTNET_RESOURCE_SERVICE_ENDPOINT_URL": "https://localhost:22299" + } + }, + "http": { + "commandName": "Project", + "dotnetRunMessages": true, + "launchBrowser": true, + "applicationUrl": "http://localhost:15161", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development", + "DOTNET_ENVIRONMENT": "Development", + "DOTNET_DASHBOARD_OTLP_ENDPOINT_URL": "http://localhost:19113", + "DOTNET_RESOURCE_SERVICE_ENDPOINT_URL": "http://localhost:20111" + } + } + } +} diff --git a/src/Exceptionless.AppHost/appsettings.Development.json b/src/Exceptionless.AppHost/appsettings.Development.json new file mode 100644 index 0000000000..0c208ae918 --- /dev/null +++ b/src/Exceptionless.AppHost/appsettings.Development.json @@ -0,0 +1,8 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.AspNetCore": "Warning" + } + } +} diff --git a/src/Exceptionless.AppHost/appsettings.json b/src/Exceptionless.AppHost/appsettings.json new file mode 100644 index 0000000000..31c092aa45 --- /dev/null +++ b/src/Exceptionless.AppHost/appsettings.json @@ -0,0 +1,9 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.AspNetCore": "Warning", + "Aspire.Hosting.Dcp": "Warning" + } + } +} diff --git a/src/Exceptionless.Core/Configuration/CacheOptions.cs b/src/Exceptionless.Core/Configuration/CacheOptions.cs index a0d788c482..3427e1a1bb 100644 --- a/src/Exceptionless.Core/Configuration/CacheOptions.cs +++ b/src/Exceptionless.Core/Configuration/CacheOptions.cs @@ -20,14 +20,29 @@ public static CacheOptions ReadFromConfiguration(IConfiguration config, AppOptio options.ScopePrefix = !String.IsNullOrEmpty(options.Scope) ? $"{options.Scope}-" : String.Empty; string? cs = config.GetConnectionString("Cache"); - options.Data = cs.ParseConnectionString(); - options.Provider = options.Data.GetString(nameof(options.Provider)); + if (cs != null) + { + options.Data = cs.ParseConnectionString(); + options.Provider = options.Data.GetString(nameof(options.Provider)); + } + else + { + var redisConnectionString = config.GetConnectionString("Redis"); + if (!String.IsNullOrEmpty(redisConnectionString)) + { + options.Provider = "redis"; + } + } string? providerConnectionString = !String.IsNullOrEmpty(options.Provider) ? config.GetConnectionString(options.Provider) : null; if (!String.IsNullOrEmpty(providerConnectionString)) - options.Data.AddRange(providerConnectionString.ParseConnectionString()); + { + var providerOptions = providerConnectionString.ParseConnectionString(defaultKey: "server"); + options.Data ??= new Dictionary(StringComparer.OrdinalIgnoreCase); + options.Data.AddRange(providerOptions); + } - options.ConnectionString = options.Data.BuildConnectionString(new HashSet { nameof(options.Provider) }); + options.ConnectionString = options.Data.BuildConnectionString([nameof(options.Provider)]); return options; } diff --git a/src/Exceptionless.Core/Configuration/CustomEnvironmentVariablesConfiguration.cs b/src/Exceptionless.Core/Configuration/CustomEnvironmentVariablesConfiguration.cs new file mode 100644 index 0000000000..f626ff6788 --- /dev/null +++ b/src/Exceptionless.Core/Configuration/CustomEnvironmentVariablesConfiguration.cs @@ -0,0 +1,56 @@ +using System.Collections; +using Microsoft.Extensions.Configuration; + +namespace Exceptionless.Core.Configuration; + +public static class CustomEnvironmentVariablesExtensions +{ + public static IConfigurationBuilder AddCustomEnvironmentVariables(this IConfigurationBuilder configurationBuilder) + { + configurationBuilder.Add(new CustomEnvironmentVariablesConfigurationSource()); + return configurationBuilder; + } +} + +public class CustomEnvironmentVariablesConfigurationSource : IConfigurationSource +{ + public IConfigurationProvider Build(IConfigurationBuilder builder) + { + return new CustomEnvironmentVariablesConfigurationProvider(); + } +} + +public class CustomEnvironmentVariablesConfigurationProvider : ConfigurationProvider +{ + public override void Load() => Load(Environment.GetEnvironmentVariables()); + + internal void Load(IDictionary envVariables) + { + var data = new Dictionary(StringComparer.OrdinalIgnoreCase); + + IDictionaryEnumerator e = envVariables.GetEnumerator(); + try + { + while (e.MoveNext()) + { + string key = (string)e.Entry.Key; + string? value = (string?)e.Entry.Value; + + var normalizedKey = Normalize(key); + // remove EX_ prefix + if (normalizedKey.StartsWith("EX_")) + data[normalizedKey.Substring(3)] = value; + else + data[normalizedKey] = value; + } + } + finally + { + (e as IDisposable)?.Dispose(); + } + + Data = data; + } + + private static string Normalize(string key) => key.Replace("__", ConfigurationPath.KeyDelimiter); +} diff --git a/src/Exceptionless.Core/Configuration/ElasticsearchOptions.cs b/src/Exceptionless.Core/Configuration/ElasticsearchOptions.cs index 38eabafe72..bce6d41cd7 100644 --- a/src/Exceptionless.Core/Configuration/ElasticsearchOptions.cs +++ b/src/Exceptionless.Core/Configuration/ElasticsearchOptions.cs @@ -51,7 +51,7 @@ public static ElasticsearchOptions ReadFromConfiguration(IConfiguration config, private static void ParseConnectionString(string? connectionString, ElasticsearchOptions options, AppMode appMode) { - var pairs = connectionString.ParseConnectionString(); + var pairs = connectionString.ParseConnectionString(defaultKey: "server"); options.ServerUrl = pairs.GetString("server", "http://localhost:9200"); int shards = pairs.GetValueOrDefault("shards", 1); diff --git a/src/Exceptionless.Core/Configuration/MessageBusOptions.cs b/src/Exceptionless.Core/Configuration/MessageBusOptions.cs index b69407028f..23327d2387 100644 --- a/src/Exceptionless.Core/Configuration/MessageBusOptions.cs +++ b/src/Exceptionless.Core/Configuration/MessageBusOptions.cs @@ -22,12 +22,27 @@ public static MessageBusOptions ReadFromConfiguration(IConfiguration config, App options.Topic = config.GetValue(nameof(options.Topic), $"{options.ScopePrefix}messages")!; string? cs = config.GetConnectionString("MessageBus"); - options.Data = cs.ParseConnectionString(); - options.Provider = options.Data.GetString(nameof(options.Provider)); + if (cs != null) + { + options.Data = cs.ParseConnectionString(); + options.Provider = options.Data.GetString(nameof(options.Provider)); + } + else + { + var redisConnectionString = config.GetConnectionString("Redis"); + if (!String.IsNullOrEmpty(redisConnectionString)) + { + options.Provider = "redis"; + } + } string? providerConnectionString = !String.IsNullOrEmpty(options.Provider) ? config.GetConnectionString(options.Provider) : null; if (!String.IsNullOrEmpty(providerConnectionString)) - options.Data.AddRange(providerConnectionString.ParseConnectionString()); + { + var providerOptions = providerConnectionString.ParseConnectionString(defaultKey: "server"); + options.Data ??= new Dictionary(StringComparer.OrdinalIgnoreCase); + options.Data.AddRange(providerOptions); + } options.ConnectionString = options.Data.BuildConnectionString(new HashSet { nameof(options.Provider) }); diff --git a/src/Exceptionless.Core/Configuration/QueueOptions.cs b/src/Exceptionless.Core/Configuration/QueueOptions.cs index 9ab058ca66..5955eb25fd 100644 --- a/src/Exceptionless.Core/Configuration/QueueOptions.cs +++ b/src/Exceptionless.Core/Configuration/QueueOptions.cs @@ -13,6 +13,8 @@ public class QueueOptions public string Scope { get; internal set; } = null!; public string ScopePrefix { get; internal set; } = null!; + public bool MetricsPollingEnabled { get; set; } = true; + public TimeSpan MetricsPollingInterval { get; set; } = TimeSpan.FromSeconds(5); public static QueueOptions ReadFromConfiguration(IConfiguration config, AppOptions appOptions) { @@ -20,15 +22,32 @@ public static QueueOptions ReadFromConfiguration(IConfiguration config, AppOptio options.ScopePrefix = !String.IsNullOrEmpty(options.Scope) ? $"{options.Scope}-" : String.Empty; string? cs = config.GetConnectionString("Queue"); - options.Data = cs.ParseConnectionString(); - options.Provider = options.Data.GetString(nameof(options.Provider)); + if (cs != null) + { + options.Data = cs.ParseConnectionString(); + options.Provider = options.Data.GetString(nameof(options.Provider)); + } + else + { + var redisConnectionString = config.GetConnectionString("Redis"); + if (!String.IsNullOrEmpty(redisConnectionString)) + { + options.Provider = "redis"; + } + } string? providerConnectionString = !String.IsNullOrEmpty(options.Provider) ? config.GetConnectionString(options.Provider) : null; if (!String.IsNullOrEmpty(providerConnectionString)) - options.Data.AddRange(providerConnectionString.ParseConnectionString()); + { + var providerOptions = providerConnectionString.ParseConnectionString(defaultKey: "server"); + options.Data ??= new Dictionary(StringComparer.OrdinalIgnoreCase); + options.Data.AddRange(providerOptions); + } options.ConnectionString = options.Data.BuildConnectionString(new HashSet { nameof(options.Provider) }); + options.MetricsPollingInterval = appOptions.AppMode == AppMode.Development ? TimeSpan.FromSeconds(15) : TimeSpan.FromSeconds(5); + return options; } } diff --git a/src/Exceptionless.Core/Configuration/StorageOptions.cs b/src/Exceptionless.Core/Configuration/StorageOptions.cs index 2c114e973d..79fbbba864 100644 --- a/src/Exceptionless.Core/Configuration/StorageOptions.cs +++ b/src/Exceptionless.Core/Configuration/StorageOptions.cs @@ -16,18 +16,31 @@ public class StorageOptions public static StorageOptions ReadFromConfiguration(IConfiguration config, AppOptions appOptions) { - var options = new StorageOptions(); - - options.Scope = appOptions.AppScope; + var options = new StorageOptions { Scope = appOptions.AppScope }; options.ScopePrefix = !String.IsNullOrEmpty(options.Scope) ? $"{options.Scope}-" : String.Empty; string? cs = config.GetConnectionString("Storage"); - options.Data = cs.ParseConnectionString(); - options.Provider = options.Data.GetString(nameof(options.Provider)); + if (cs != null) + { + options.Data = cs.ParseConnectionString(); + options.Provider = options.Data.GetString(nameof(options.Provider)); + } + else + { + var minioConnectionString = config.GetConnectionString("S3"); + if (!String.IsNullOrEmpty(minioConnectionString)) + { + options.Provider = "s3"; + } + } string? providerConnectionString = !String.IsNullOrEmpty(options.Provider) ? config.GetConnectionString(options.Provider) : null; if (!String.IsNullOrEmpty(providerConnectionString)) - options.Data.AddRange(providerConnectionString.ParseConnectionString()); + { + var providerOptions = providerConnectionString.ParseConnectionString(defaultKey: "server"); + options.Data ??= new Dictionary(StringComparer.OrdinalIgnoreCase); + options.Data.AddRange(providerOptions); + } options.ConnectionString = options.Data.BuildConnectionString(new HashSet { nameof(options.Provider) }); diff --git a/src/Exceptionless.Core/Exceptionless.Core.csproj b/src/Exceptionless.Core/Exceptionless.Core.csproj index bf3a748b3c..4b2da10210 100644 --- a/src/Exceptionless.Core/Exceptionless.Core.csproj +++ b/src/Exceptionless.Core/Exceptionless.Core.csproj @@ -22,8 +22,8 @@ - - + + diff --git a/src/Exceptionless.Insulation/Bootstrapper.cs b/src/Exceptionless.Insulation/Bootstrapper.cs index dd690ec6e4..d47c6916b6 100644 --- a/src/Exceptionless.Insulation/Bootstrapper.cs +++ b/src/Exceptionless.Insulation/Bootstrapper.cs @@ -183,22 +183,22 @@ private static void RegisterQueue(IServiceCollection container, QueueOptions opt private static void RegisterStorage(IServiceCollection container, StorageOptions options) { - if (String.Equals(options.Provider, "aliyun")) + if (String.Equals(options.Provider, "azurestorage")) { - container.ReplaceSingleton(s => new AliyunFileStorage(new AliyunFileStorageOptions + container.ReplaceSingleton(s => new AzureFileStorage(new AzureFileStorageOptions { ConnectionString = options.ConnectionString, + ContainerName = $"{options.ScopePrefix}ex-events", Serializer = s.GetRequiredService(), TimeProvider = s.GetRequiredService(), LoggerFactory = s.GetRequiredService() })); } - else if (String.Equals(options.Provider, "azurestorage")) + else if (String.Equals(options.Provider, "aliyun")) { - container.ReplaceSingleton(s => new AzureFileStorage(new AzureFileStorageOptions + container.ReplaceSingleton(s => new AliyunFileStorage(new AliyunFileStorageOptions { ConnectionString = options.ConnectionString, - ContainerName = $"{options.ScopePrefix}ex-events", Serializer = s.GetRequiredService(), TimeProvider = s.GetRequiredService(), LoggerFactory = s.GetRequiredService() @@ -207,13 +207,21 @@ private static void RegisterStorage(IServiceCollection container, StorageOptions else if (String.Equals(options.Provider, "folder")) { string path = options.Data.GetString("path", "|DataDirectory|\\storage"); - container.AddSingleton(s => new FolderFileStorage(new FolderFileStorageOptions + container.AddSingleton(s => { - Folder = PathHelper.ExpandPath(path), - Serializer = s.GetRequiredService(), - TimeProvider = s.GetRequiredService(), - LoggerFactory = s.GetRequiredService() - })); + IFileStorage storage = new FolderFileStorage(new FolderFileStorageOptions + { + Folder = PathHelper.ExpandPath(path), + Serializer = s.GetRequiredService(), + TimeProvider = s.GetRequiredService(), + LoggerFactory = s.GetRequiredService() + }); + + if (!String.IsNullOrWhiteSpace(options.Scope)) + storage = new ScopedFileStorage(storage, options.Scope); + + return storage; + }); } else if (String.Equals(options.Provider, "minio")) { @@ -227,16 +235,14 @@ private static void RegisterStorage(IServiceCollection container, StorageOptions } else if (String.Equals(options.Provider, "s3")) { - container.ReplaceSingleton(s => new S3FileStorage(new S3FileStorageOptions - { - ConnectionString = options.ConnectionString, - Credentials = GetAWSCredentials(options.Data), - Region = GetAWSRegionEndpoint(options.Data), - Bucket = $"{options.ScopePrefix}{options.Data.GetString("bucket", "ex-events")}", - Serializer = s.GetRequiredService(), - TimeProvider = s.GetRequiredService(), - LoggerFactory = s.GetRequiredService() - })); + container.ReplaceSingleton(s => new S3FileStorage(o => o + .ConnectionString(options.ConnectionString) + .Credentials(GetAWSCredentials(options.Data)) + .Region(GetAWSRegionEndpoint(options.Data)) + .Bucket(options.Data.GetString("bucket", $"{options.ScopePrefix}ex-events")) + .Serializer(s.GetRequiredService()) + .TimeProvider(s.GetRequiredService()) + .LoggerFactory(s.GetRequiredService()))); } } @@ -251,7 +257,9 @@ private static IQueue CreateAzureStorageQueue(IServiceProvider container, WorkItemTimeout = workItemTimeout.GetValueOrDefault(TimeSpan.FromMinutes(5.0)), Serializer = container.GetRequiredService(), TimeProvider = container.GetRequiredService(), - LoggerFactory = container.GetRequiredService() + LoggerFactory = container.GetRequiredService(), + MetricsPollingEnabled = options.MetricsPollingEnabled, + MetricsPollingInterval = options.MetricsPollingInterval }); } @@ -267,7 +275,9 @@ private static IQueue CreateRedisQueue(IServiceProvider container, QueueOp RunMaintenanceTasks = runMaintenanceTasks, Serializer = container.GetRequiredService(), TimeProvider = container.GetRequiredService(), - LoggerFactory = container.GetRequiredService() + LoggerFactory = container.GetRequiredService(), + MetricsPollingEnabled = options.MetricsPollingEnabled, + MetricsPollingInterval = options.MetricsPollingInterval }); } @@ -295,7 +305,9 @@ private static IQueue CreateSQSQueue(IServiceProvider container, QueueOpti WorkItemTimeout = workItemTimeout.GetValueOrDefault(TimeSpan.FromMinutes(5.0)), Serializer = container.GetRequiredService(), TimeProvider = container.GetRequiredService(), - LoggerFactory = container.GetRequiredService() + LoggerFactory = container.GetRequiredService(), + MetricsPollingEnabled = options.MetricsPollingEnabled, + MetricsPollingInterval = options.MetricsPollingInterval }); } diff --git a/src/Exceptionless.Insulation/Exceptionless.Insulation.csproj b/src/Exceptionless.Insulation/Exceptionless.Insulation.csproj index 94aca8e88f..953d5aedea 100644 --- a/src/Exceptionless.Insulation/Exceptionless.Insulation.csproj +++ b/src/Exceptionless.Insulation/Exceptionless.Insulation.csproj @@ -14,10 +14,10 @@ - - + + - + diff --git a/src/Exceptionless.Job/Exceptionless.Job.csproj b/src/Exceptionless.Job/Exceptionless.Job.csproj index 5ae38e689a..a7161b3e28 100644 --- a/src/Exceptionless.Job/Exceptionless.Job.csproj +++ b/src/Exceptionless.Job/Exceptionless.Job.csproj @@ -7,7 +7,7 @@ - + @@ -15,12 +15,12 @@ - + - + - - + + diff --git a/src/Exceptionless.Job/Program.cs b/src/Exceptionless.Job/Program.cs index 80da975ef5..2433bb65be 100644 --- a/src/Exceptionless.Job/Program.cs +++ b/src/Exceptionless.Job/Program.cs @@ -1,5 +1,6 @@ -using System.Diagnostics; +using System.Diagnostics; using Exceptionless.Core; +using Exceptionless.Core.Configuration; using Exceptionless.Core.Extensions; using Exceptionless.Core.Jobs; using Exceptionless.Core.Jobs.Elastic; @@ -49,8 +50,7 @@ public static IHostBuilder CreateHostBuilder(string[] args) .SetBasePath(Directory.GetCurrentDirectory()) .AddYamlFile("appsettings.yml", optional: true, reloadOnChange: true) .AddYamlFile($"appsettings.{environment}.yml", optional: true, reloadOnChange: true) - .AddEnvironmentVariables("EX_") - .AddEnvironmentVariables("ASPNETCORE_") + .AddCustomEnvironmentVariables() .AddCommandLine(args) .Build(); @@ -60,6 +60,9 @@ public static IHostBuilder CreateHostBuilder(string[] args) .ForContext(); var options = AppOptions.ReadFromConfiguration(config); + // only poll the queue metrics if this process is going to run the stack event count job + options.QueueOptions.MetricsPollingEnabled = jobOptions.StackEventCount; + var apmConfig = new ApmConfig(config, $"job-{jobOptions.JobName.ToLowerUnderscoredWords('-')}", options.InformationalVersion, options.CacheOptions.Provider == "redis"); Log.Information("Bootstrapping Exceptionless {JobName} job(s) in {AppMode} mode ({InformationalVersion}) on {MachineName} with options {@Options}", jobOptions.JobName ?? "All", environment, options.InformationalVersion, Environment.MachineName, options); @@ -85,13 +88,13 @@ public static IHostBuilder CreateHostBuilder(string[] args) app.UseSerilogRequestLogging(o => { o.MessageTemplate = "TraceId={TraceId} HTTP {RequestMethod} {RequestPath} responded {StatusCode} in {Elapsed:0.0000} ms"; - o.GetLevel = (context, duration, ex) => + o.GetLevel = new Func((context, duration, ex) => { if (ex is not null || context.Response.StatusCode > 499) return LogEventLevel.Error; return duration < 1000 && context.Response.StatusCode < 400 ? LogEventLevel.Debug : LogEventLevel.Information; - }; + }); }); Bootstrapper.LogConfiguration(app.ApplicationServices, options, app.ApplicationServices.GetRequiredService>()); @@ -146,27 +149,27 @@ private static void AddJobs(IServiceCollection services, JobRunnerOptions option services.AddJob(); if (options.CloseInactiveSessions) - services.AddJob(o => o.WaitForStartupActions(true)); + services.AddJob(o => o.WaitForStartupActions()); if (options.DailySummary) - services.AddJob(o => o.WaitForStartupActions(true)); + services.AddJob(o => o.WaitForStartupActions()); if (options.DataMigration) - services.AddJob(o => o.WaitForStartupActions(true)); + services.AddJob(o => o.WaitForStartupActions()); if (options is { DownloadGeoIPDatabase: true, AllJobs: true }) services.AddCronJob("0 1 * * *"); if (options is { DownloadGeoIPDatabase: true, AllJobs: false }) - services.AddJob(o => o.WaitForStartupActions(true)); + services.AddJob(o => o.WaitForStartupActions()); if (options.EventNotifications) - services.AddJob(o => o.WaitForStartupActions(true)); + services.AddJob(o => o.WaitForStartupActions()); if (options.EventPosts) - services.AddJob(o => o.WaitForStartupActions(true)); + services.AddJob(o => o.WaitForStartupActions()); if (options.EventUsage) - services.AddJob(o => o.WaitForStartupActions(true)); + services.AddJob(o => o.WaitForStartupActions()); if (options.EventUserDescriptions) - services.AddJob(o => o.WaitForStartupActions(true)); + services.AddJob(o => o.WaitForStartupActions()); if (options.MailMessage) - services.AddJob(o => o.WaitForStartupActions(true)); + services.AddJob(o => o.WaitForStartupActions()); if (options is { MaintainIndexes: true, AllJobs: true }) services.AddCronJob("10 */2 * * *"); @@ -174,14 +177,14 @@ private static void AddJobs(IServiceCollection services, JobRunnerOptions option services.AddJob(); if (options.Migration) - services.AddJob(o => o.WaitForStartupActions(true)); + services.AddJob(o => o.WaitForStartupActions()); if (options.StackStatus) - services.AddJob(o => o.WaitForStartupActions(true)); + services.AddJob(o => o.WaitForStartupActions()); if (options.StackEventCount) - services.AddJob(o => o.WaitForStartupActions(true)); + services.AddJob(o => o.WaitForStartupActions()); if (options.WebHooks) - services.AddJob(o => o.WaitForStartupActions(true)); + services.AddJob(o => o.WaitForStartupActions()); if (options.WorkItem) - services.AddJob(o => o.WaitForStartupActions(true)); + services.AddJob(o => o.WaitForStartupActions()); } } diff --git a/src/Exceptionless.Job/Properties/launchSettings.json b/src/Exceptionless.Job/Properties/launchSettings.json index 9043c97165..f6f61e578f 100644 --- a/src/Exceptionless.Job/Properties/launchSettings.json +++ b/src/Exceptionless.Job/Properties/launchSettings.json @@ -5,7 +5,7 @@ "environmentVariables": { "EX_AppMode": "Development" }, - "launchBrowser": true, + "launchBrowser": false, "applicationUrl": "https://localhost:5002;http://localhost:5003" }, "CleanupDataJob": { @@ -14,7 +14,7 @@ "environmentVariables": { "EX_AppMode": "Development" }, - "launchBrowser": true, + "launchBrowser": false, "applicationUrl": "https://localhost:5002;http://localhost:5003" }, "DataMigrationJob": { @@ -23,7 +23,7 @@ "environmentVariables": { "EX_AppMode": "Development" }, - "launchBrowser": true, + "launchBrowser": false, "applicationUrl": "https://localhost:5002;http://localhost:5003" }, "MigrationJob": { @@ -32,7 +32,7 @@ "environmentVariables": { "EX_AppMode": "Development" }, - "launchBrowser": true, + "launchBrowser": false, "applicationUrl": "https://localhost:5002;http://localhost:5003" }, "EventPostsJob": { @@ -41,7 +41,7 @@ "environmentVariables": { "EX_AppMode": "Development" }, - "launchBrowser": true, + "launchBrowser": false, "applicationUrl": "https://localhost:5002;http://localhost:5003" }, "EventUserDescriptionsJob": { @@ -50,7 +50,7 @@ "environmentVariables": { "EX_AppMode": "Development" }, - "launchBrowser": true, + "launchBrowser": false, "applicationUrl": "https://localhost:5002;http://localhost:5003" }, "EventNotificationsJob": { @@ -59,7 +59,7 @@ "environmentVariables": { "EX_AppMode": "Development" }, - "launchBrowser": true, + "launchBrowser": false, "applicationUrl": "https://localhost:5002;http://localhost:5003" }, "MailMessageJob": { @@ -68,7 +68,7 @@ "environmentVariables": { "EX_AppMode": "Development" }, - "launchBrowser": true, + "launchBrowser": false, "applicationUrl": "https://localhost:5002;http://localhost:5003" }, "WebHooksJob": { @@ -77,7 +77,7 @@ "environmentVariables": { "EX_AppMode": "Development" }, - "launchBrowser": true, + "launchBrowser": false, "applicationUrl": "https://localhost:5002;http://localhost:5003" }, "CloseInactiveSessionsJob": { @@ -86,7 +86,7 @@ "environmentVariables": { "EX_AppMode": "Development" }, - "launchBrowser": true, + "launchBrowser": false, "applicationUrl": "https://localhost:5002;http://localhost:5003" }, "DailySummaryJob": { @@ -95,7 +95,7 @@ "environmentVariables": { "EX_AppMode": "Development" }, - "launchBrowser": true, + "launchBrowser": false, "applicationUrl": "https://localhost:5002;http://localhost:5003" }, "DownloadGeoIPDatabaseJob": { @@ -104,7 +104,7 @@ "environmentVariables": { "EX_AppMode": "Development" }, - "launchBrowser": true, + "launchBrowser": false, "applicationUrl": "https://localhost:5002;http://localhost:5003" }, "WorkItemJob": { @@ -113,7 +113,7 @@ "environmentVariables": { "EX_AppMode": "Development" }, - "launchBrowser": true, + "launchBrowser": false, "applicationUrl": "https://localhost:5002;http://localhost:5003" }, "MaintainIndexesJob": { @@ -122,7 +122,7 @@ "environmentVariables": { "EX_AppMode": "Development" }, - "launchBrowser": true, + "launchBrowser": false, "applicationUrl": "https://localhost:5002;http://localhost:5003" }, "StackEventCountJob": { @@ -131,7 +131,7 @@ "environmentVariables": { "EX_AppMode": "Development" }, - "launchBrowser": true, + "launchBrowser": false, "applicationUrl": "https://localhost:5002;http://localhost:5003" }, "EventSnapshotJob": { @@ -140,8 +140,8 @@ "environmentVariables": { "EX_AppMode": "Development" }, - "launchBrowser": true, + "launchBrowser": false, "applicationUrl": "https://localhost:5002;http://localhost:5003" } } -} \ No newline at end of file +} diff --git a/src/Exceptionless.Job/appsettings.Development.yml b/src/Exceptionless.Job/appsettings.Development.yml index 438c3a12c0..bdd125522a 100644 --- a/src/Exceptionless.Job/appsettings.Development.yml +++ b/src/Exceptionless.Job/appsettings.Development.yml @@ -5,7 +5,7 @@ ConnectionStrings: # Cache: provider=redis; # MessageBus: provider=redis; # Queue: provider=redis; - Storage: provider=folder;path=..\Exceptionless.Web\storage +# Storage: provider=folder;path=..\Exceptionless.Web\storage Email: smtp://localhost:1025 # Base url for the ui used to build links in emails and other places. diff --git a/src/Exceptionless.Web/ApmExtensions.cs b/src/Exceptionless.Web/ApmExtensions.cs index 3c96da994d..ef11a669f2 100644 --- a/src/Exceptionless.Web/ApmExtensions.cs +++ b/src/Exceptionless.Web/ApmExtensions.cs @@ -36,7 +36,7 @@ public static IHostBuilder AddApm(this IHostBuilder builder, ApmConfig config) b.AddAspNetCoreInstrumentation(o => { - o.Filter = context => + o.Filter = new Func(context => { if (context.Request.Path.StartsWithSegments("/api/v2/push", StringComparison.OrdinalIgnoreCase)) return false; @@ -48,7 +48,7 @@ public static IHostBuilder AddApm(this IHostBuilder builder, ApmConfig config) return false; return true; - }; + }); }); b.AddElasticsearchClientInstrumentation(c => @@ -129,6 +129,7 @@ public static IHostBuilder AddApm(this IHostBuilder builder, ApmConfig config) b.AddHttpClientInstrumentation(); b.AddAspNetCoreInstrumentation(); b.AddMeter("Exceptionless", "Foundatio"); + b.AddMeter("System.Runtime"); b.AddRuntimeInstrumentation(); b.AddProcessInstrumentation(); diff --git a/src/Exceptionless.Web/ClientApp/vite.config.ts b/src/Exceptionless.Web/ClientApp/vite.config.ts index f366b5d315..ade7bab6fb 100644 --- a/src/Exceptionless.Web/ClientApp/vite.config.ts +++ b/src/Exceptionless.Web/ClientApp/vite.config.ts @@ -17,7 +17,8 @@ export default defineConfig({ ], server: { hmr: aspNetConfig.hmr, - port: 5173, + host: true, + port: parseInt(process.env.PORT ?? '5173'), proxy: { '/_framework': { changeOrigin: true, @@ -77,7 +78,7 @@ function getAspNetConfig() { // get current aspnetcore port / url const aspnetHttpsPort = process.env.ASPNETCORE_HTTPS_PORT; - const aspnetUrls = process.env.ASPNETCORE_URLS; + const aspnetUrls = process.env.ASPNETCORE_URLS ?? process.env.services__Api__0; const serverPort = 5173; const hmrRemoteHost = codespaceName ? `${codespaceName}-${serverPort}.${codespaceDomain}` : 'localhost'; diff --git a/src/Exceptionless.Web/Controllers/Base/ExceptionlessApiController.cs b/src/Exceptionless.Web/Controllers/Base/ExceptionlessApiController.cs index 6292a71153..fb968c017a 100644 --- a/src/Exceptionless.Web/Controllers/Base/ExceptionlessApiController.cs +++ b/src/Exceptionless.Web/Controllers/Base/ExceptionlessApiController.cs @@ -94,6 +94,16 @@ protected int GetSkip(int currentPage, int limit) return skip; } + /// + /// This call will throw an exception if the user is a token auth type. + /// This is less than ideal, and we should refactor this to be a nullable user. + /// NOTE: The only endpoints that allow token auth types is + /// - post event + /// - post user event description + /// - post session heartbeat + /// - post session end + /// - project config + /// protected virtual User CurrentUser => Request.GetUser(); protected bool CanAccessOrganization(string organizationId) @@ -101,7 +111,6 @@ protected bool CanAccessOrganization(string organizationId) return Request.CanAccessOrganization(organizationId); } - protected bool IsInOrganization([NotNullWhen(true)] string? organizationId) { if (String.IsNullOrEmpty(organizationId)) diff --git a/src/Exceptionless.Web/Controllers/EventController.cs b/src/Exceptionless.Web/Controllers/EventController.cs index 2f7b90c7c0..ca35362fb8 100644 --- a/src/Exceptionless.Web/Controllers/EventController.cs +++ b/src/Exceptionless.Web/Controllers/EventController.cs @@ -248,8 +248,8 @@ private async Task> CountInternalAsync(AppFilter sf, T } catch (Exception ex) { - using (_logger.BeginScope(new ExceptionlessState().Property("Search Filter", new { SystemFilter = sf, UserFilter = filter, Time = ti, Aggregations = aggregations }).Tag("Search").Identity(CurrentUser.EmailAddress).Property("User", CurrentUser).SetHttpContext(HttpContext))) - _logger.LogError(ex, "An error has occurred. Please check your filter or aggregations"); + using var _ = _logger.BeginScope(new ExceptionlessState().Property("Search Filter", new { SystemFilter = sf, UserFilter = filter, Time = ti, Aggregations = aggregations }).Tag("Search").Identity(CurrentUser.EmailAddress).Property("User", CurrentUser).SetHttpContext(HttpContext)); + _logger.LogError(ex, "An error has occurred. Please check your filter or aggregations: {Message}", ex.Message); throw; } @@ -867,8 +867,8 @@ await Task.WhenAll( { if (projectId != _appOptions.InternalProjectId) { - using (_logger.BeginScope(new ExceptionlessState().Project(projectId).Identity(CurrentUser.EmailAddress).Property("User", CurrentUser).Property("Id", id).Property("Close", close).SetHttpContext(HttpContext))) - _logger.LogError(ex, "Error enqueuing session heartbeat"); + using var _ = _logger.BeginScope(new ExceptionlessState().Project(projectId).Property("Id", id).Property("Close", close).SetHttpContext(HttpContext)); + _logger.LogError(ex, "Error enqueuing session heartbeat: {Message}", ex.Message); } throw; @@ -1127,8 +1127,8 @@ await _eventPostService.EnqueueAsync(new EventPost(_appOptions.EnableArchive) { if (projectId != _appOptions.InternalProjectId) { - using (_logger.BeginScope(new ExceptionlessState().Project(projectId).Identity(CurrentUser.EmailAddress).Property("User", CurrentUser).SetHttpContext(HttpContext))) - _logger.LogError(ex, "Error enqueuing event post"); + using var _ = _logger.BeginScope(new ExceptionlessState().Project(projectId).SetHttpContext(HttpContext)); + _logger.LogError(ex, "Error enqueuing event post: {Message}", ex.Message); } throw; @@ -1328,8 +1328,8 @@ await _eventPostService.EnqueueAsync(new EventPost(_appOptions.EnableArchive) { if (projectId != _appOptions.InternalProjectId) { - using (_logger.BeginScope(new ExceptionlessState().Project(projectId).Identity(CurrentUser.EmailAddress).Property("User", CurrentUser).SetHttpContext(HttpContext))) - _logger.LogError(ex, "Error enqueuing event post"); + using var _ = _logger.BeginScope(new ExceptionlessState().Project(projectId).SetHttpContext(HttpContext)); + _logger.LogError(ex, "Error enqueuing event post: {Message}", ex.Message); } throw; diff --git a/src/Exceptionless.Web/Exceptionless.Web.csproj b/src/Exceptionless.Web/Exceptionless.Web.csproj index dbd09aebad..7ca6504b29 100644 --- a/src/Exceptionless.Web/Exceptionless.Web.csproj +++ b/src/Exceptionless.Web/Exceptionless.Web.csproj @@ -1,12 +1,4 @@ - - ClientApp\ - ClientApp.angular\ - $(DefaultItemExcludes);$(SpaRoot)node_modules\**;$(AngularSpaRoot)node_modules\**; - http://localhost:5173/next - npm run dev - false - @@ -24,8 +16,8 @@ - - + + @@ -33,13 +25,13 @@ - + - + - - - + + + @@ -63,39 +55,4 @@ - - - - - - - - - - - - - - - - - - - - - - wwwroot\next\%(RecursiveDir)%(FileName)%(Extension) - Always - true - - - - - - wwwroot\%(RecursiveDir)%(FileName)%(Extension) - Always - true - - - diff --git a/src/Exceptionless.Web/Program.cs b/src/Exceptionless.Web/Program.cs index ef2fa06082..88bb1f0dad 100644 --- a/src/Exceptionless.Web/Program.cs +++ b/src/Exceptionless.Web/Program.cs @@ -1,5 +1,6 @@ using System.Diagnostics; using Exceptionless.Core; +using Exceptionless.Core.Configuration; using Exceptionless.Core.Extensions; using Exceptionless.Insulation.Configuration; using OpenTelemetry; @@ -43,8 +44,7 @@ public static IHostBuilder CreateHostBuilder(string[] args) .SetBasePath(Directory.GetCurrentDirectory()) .AddYamlFile("appsettings.yml", optional: true, reloadOnChange: true) .AddYamlFile($"appsettings.{environment}.yml", optional: true, reloadOnChange: true) - .AddEnvironmentVariables("EX_") - .AddEnvironmentVariables("ASPNETCORE_") + .AddCustomEnvironmentVariables() .AddCommandLine(args) .Build(); @@ -61,6 +61,9 @@ public static IHostBuilder CreateHostBuilder(IConfigurationRoot config, string e .ForContext(); var options = AppOptions.ReadFromConfiguration(config); + // only poll the queue metrics if this process is going to host the jobs + options.QueueOptions.MetricsPollingEnabled = options.RunJobsInProcess; + var apmConfig = new ApmConfig(config, "web", options.InformationalVersion, options.CacheOptions.Provider == "redis"); Log.Information("Bootstrapping Exceptionless Web in {AppMode} mode ({InformationalVersion}) on {MachineName} with options {@Options}", environment, options.InformationalVersion, Environment.MachineName, options); diff --git a/src/Exceptionless.Web/Properties/launchSettings.json b/src/Exceptionless.Web/Properties/launchSettings.json index c660ee9846..a62987a549 100644 --- a/src/Exceptionless.Web/Properties/launchSettings.json +++ b/src/Exceptionless.Web/Properties/launchSettings.json @@ -2,19 +2,7 @@ "profiles": { "Exceptionless": { "commandName": "Project", - "launchBrowser": true, - "launchUrl": "http://localhost:5200/next", - "environmentVariables": { - "EX_AppMode": "Development", - "ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy", - "OTEL_EXPORTER_OTLP_ENDPOINT": "http://localhost:8200" - }, - "dotnetRunMessages": true, - "applicationUrl": "http://localhost:5200" - }, - "Exceptionless API": { - "commandName": "Project", - "launchBrowser": true, + "launchBrowser": false, "environmentVariables": { "EX_AppMode": "Development" }, diff --git a/src/Exceptionless.Web/Startup.cs b/src/Exceptionless.Web/Startup.cs index a2bd162f63..c20655cd01 100644 --- a/src/Exceptionless.Web/Startup.cs +++ b/src/Exceptionless.Web/Startup.cs @@ -14,6 +14,7 @@ using Foundatio.Extensions.Hosting.Startup; using Foundatio.Repositories.Exceptions; using Joonasw.AspNetCore.SecurityHeaders; +using Joonasw.AspNetCore.SecurityHeaders.Csp; using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Diagnostics.HealthChecks; using Microsoft.AspNetCore.Hosting.Server.Features; @@ -299,18 +300,19 @@ ApplicationException applicationException when applicationException.Message.Cont .To("https://api-iam.intercom.io/") .To("wss://nexus-websocket-a.intercom.io"); - csp.OnSendingHeader = context => + csp.OnSendingHeader = new Func(context => { context.ShouldNotSend = context.HttpContext.Request.Path.StartsWithSegments("/api"); return Task.CompletedTask; - }; + }); }); app.UseSerilogRequestLogging(o => { o.EnrichDiagnosticContext = (context, httpContext) => { - context.Set("ActivityId", Activity.Current?.Id); + if (Activity.Current?.Id is not null) + context.Set("ActivityId", Activity.Current.Id); }; o.MessageTemplate = "{ActivityId} HTTP {RequestMethod} {RequestPath} responded {StatusCode} in {Elapsed:0.0000} ms"; o.GetLevel = (context, duration, ex) => diff --git a/src/Exceptionless.Web/appsettings.Development.yml b/src/Exceptionless.Web/appsettings.Development.yml index a399826abe..2abf77f96e 100644 --- a/src/Exceptionless.Web/appsettings.Development.yml +++ b/src/Exceptionless.Web/appsettings.Development.yml @@ -5,7 +5,7 @@ ConnectionStrings: # Cache: provider=redis; # MessageBus: provider=redis; # Queue: provider=redis; - Storage: provider=folder;path=.\storage +# Storage: provider=folder;path=.\storage # LDAP: '' # Email: smtp://localhost:1025 diff --git a/start-services.ps1 b/start-services.ps1 deleted file mode 100644 index 559514bd2b..0000000000 --- a/start-services.ps1 +++ /dev/null @@ -1 +0,0 @@ -docker compose -f docker/docker-compose.yml up --detach elasticsearch kibana \ No newline at end of file diff --git a/stop-services.ps1 b/stop-services.ps1 deleted file mode 100644 index e407f79689..0000000000 --- a/stop-services.ps1 +++ /dev/null @@ -1 +0,0 @@ -docker compose -f docker/docker-compose.yml down --remove-orphans \ No newline at end of file diff --git a/tests/Exceptionless.Tests/AppWebHostFactory.cs b/tests/Exceptionless.Tests/AppWebHostFactory.cs index 78ce6a99e8..7b9abeb862 100644 --- a/tests/Exceptionless.Tests/AppWebHostFactory.cs +++ b/tests/Exceptionless.Tests/AppWebHostFactory.cs @@ -1,12 +1,36 @@ +using Aspire.Hosting; +using Aspire.Hosting.ApplicationModel; using Exceptionless.Insulation.Configuration; using Exceptionless.Web; using Microsoft.AspNetCore.Mvc.Testing; using Microsoft.AspNetCore.TestHost; +using Xunit; namespace Exceptionless.Tests; -public class AppWebHostFactory : WebApplicationFactory +public class AppWebHostFactory : WebApplicationFactory, IAsyncLifetime { + private DistributedApplication? _app; + + public DistributedApplication App => _app ?? throw new InvalidOperationException("The application is not initialized"); + + public Task InitializeAsync() + { + var options = new DistributedApplicationOptions { AssemblyName = typeof(ElasticsearchResource).Assembly.FullName, DisableDashboard = true }; + var builder = DistributedApplication.CreateBuilder(options); + + // don't use random ports for tests + builder.Configuration["DcpPublisher:RandomizePorts"] = "false"; + + builder.AddElasticsearch("Elasticsearch", port: 9200) + .WithContainerName("Exceptionless-Elasticsearch-Test") + .WithLifetime(ContainerLifetime.Persistent); + + _app = builder.Build(); + + return _app.StartAsync(); + } + protected override void ConfigureWebHost(IWebHostBuilder builder) { builder.UseSolutionRelativeContentRoot("src/Exceptionless.Web"); @@ -21,4 +45,12 @@ protected override IHostBuilder CreateHostBuilder() return Program.CreateHostBuilder(config, Environments.Development); } + + async Task IAsyncLifetime.DisposeAsync() + { + if (_app is not null) + { + await _app.DisposeAsync(); + } + } } diff --git a/tests/Exceptionless.Tests/Exceptionless.Tests.csproj b/tests/Exceptionless.Tests/Exceptionless.Tests.csproj index d6ebf316ba..e18e4f4f01 100644 --- a/tests/Exceptionless.Tests/Exceptionless.Tests.csproj +++ b/tests/Exceptionless.Tests/Exceptionless.Tests.csproj @@ -9,18 +9,20 @@ + - + runtime; build; native; contentfiles; analyzers; buildtransitive all + diff --git a/tests/Exceptionless.Tests/TestWithServices.cs b/tests/Exceptionless.Tests/TestWithServices.cs index cc6e124af6..40011b3954 100644 --- a/tests/Exceptionless.Tests/TestWithServices.cs +++ b/tests/Exceptionless.Tests/TestWithServices.cs @@ -1,5 +1,6 @@ using Exceptionless.Core; using Exceptionless.Core.Authentication; +using Exceptionless.Core.Configuration; using Exceptionless.Core.Extensions; using Exceptionless.Core.Mail; using Exceptionless.Helpers; @@ -17,11 +18,10 @@ namespace Exceptionless.Tests; -public class TestWithServices : TestWithLoggingBase, IAsyncLifetime +public class TestWithServices : TestWithLoggingBase, IDisposable { private readonly IServiceProvider _container; private readonly ProxyTimeProvider _timeProvider; - private static bool _startupActionsRun; public TestWithServices(ITestOutputHelper output) : base(output) { @@ -37,18 +37,6 @@ public TestWithServices(ITestOutputHelper output) : base(output) else throw new InvalidOperationException("TimeProvider must be of type ProxyTimeProvider"); } - - public virtual async Task InitializeAsync() - { - if (_startupActionsRun) - return; - - var result = await _container.RunStartupActionsAsync(); - if (!result.Success) - throw new ApplicationException($"Startup action \"{result.FailedActionName}\" failed"); - - _startupActionsRun = true; - } protected ProxyTimeProvider TimeProvider => _timeProvider; protected TService GetService() where TService : class @@ -83,7 +71,7 @@ private IServiceProvider CreateContainer() var config = new ConfigurationBuilder() .SetBasePath(AppContext.BaseDirectory) .AddYamlFile("appsettings.yml", optional: false, reloadOnChange: false) - .AddEnvironmentVariables() + .AddCustomEnvironmentVariables() .Build(); services.AddSingleton(config); @@ -94,9 +82,8 @@ private IServiceProvider CreateContainer() return services.BuildServiceProvider(); } - public Task DisposeAsync() + public void Dispose() { _timeProvider.Restore(); - return Task.CompletedTask; } }