-
-
Notifications
You must be signed in to change notification settings - Fork 516
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* Working on migrating to aspire * Progress * Some aspire progress * Fix bad merge * Update to Aspire 9 * Fix duplicate project refs * Update elasticsearch to 8.16.1 * Add storage to Aspire * Update Elasticsearch * Fix tests * Revert some changes. Fix linting. * Revert more changes * Cleanup * Use the right Elasticsearch docker image * Use explicit minio version * Fixed launch setting * Removed start and stop services * Use fixed web client ports * Use S3 storage when running local * Fixed an issue where code could throw due to CurrentUser * Fix S3 * [BREAKING] Remove scope prefix from bucket names and instead use scoped file storage for app scopes * Only poll queue metrics in the same process that is running the stack event count job * Reverted some of the breaking changes around storage. --------- Co-authored-by: Blake Niemyjski <[email protected]>
- Loading branch information
Showing
42 changed files
with
924 additions
and
210 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,27 @@ | ||
| <Project Sdk="Microsoft.NET.Sdk"> | ||
|
|
||
| <Sdk Name="Aspire.AppHost.Sdk" Version="9.0.0" /> | ||
|
|
||
| <PropertyGroup> | ||
| <OutputType>Exe</OutputType> | ||
| <TargetFramework>net9.0</TargetFramework> | ||
| <ImplicitUsings>enable</ImplicitUsings> | ||
| <Nullable>enable</Nullable> | ||
| <IsAspireHost>true</IsAspireHost> | ||
| <UserSecretsId>a9c2ddcc-e51d-4cd1-9782-96e1d74eec87</UserSecretsId> | ||
| </PropertyGroup> | ||
|
|
||
| <ItemGroup> | ||
| <PackageReference Include="Aspire.Hosting.AppHost" Version="9.0.0" /> | ||
| <PackageReference Include="Aspire.Hosting.NodeJs" Version="9.0.0" /> | ||
| <PackageReference Include="Aspire.Hosting.Redis" Version="9.0.0" /> | ||
| <PackageReference Include="AspNetCore.HealthChecks.Elasticsearch" Version="8.0.1" /> | ||
| <PackageReference Include="Foundatio.AWS" Version="11.0.6" /> | ||
| </ItemGroup> | ||
|
|
||
| <ItemGroup> | ||
| <ProjectReference Include="..\Exceptionless.Job\Exceptionless.Job.csproj" /> | ||
| <ProjectReference Include="..\Exceptionless.Web\Exceptionless.Web.csproj" /> | ||
| </ItemGroup> | ||
|
|
||
| </Project> |
122 changes: 122 additions & 0 deletions
122
src/Exceptionless.AppHost/Extensions/ElasticsearchExtensions.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,122 @@ | ||
| using Aspire.Hosting.Lifecycle; | ||
| using Aspire.Hosting.Utils; | ||
| using HealthChecks.Elasticsearch; | ||
| using Microsoft.Extensions.DependencyInjection; | ||
| using Microsoft.Extensions.Diagnostics.HealthChecks; | ||
|
|
||
| namespace Aspire.Hosting; | ||
|
|
||
| /// <summary> | ||
| /// Provides extension methods for adding Elasticsearch resources to the application model. | ||
| /// </summary> | ||
| public static class ElasticsearchBuilderExtensions | ||
| { | ||
| private const int ElasticsearchPort = 9200; | ||
| private const int ElasticsearchInternalPort = 9300; | ||
| private const int KibanaPort = 5601; | ||
|
|
||
| /// <summary> | ||
| /// Adds a Elasticsearch container to the application model. The default image is "docker.elastic.co/elasticsearch/elasticsearch". This version the package defaults to the 8.17.0 tag of the Elasticsearch container image | ||
| /// </summary> | ||
| /// <param name="builder">The <see cref="IDistributedApplicationBuilder"/>.</param> | ||
| /// <param name="name">The name of the resource. This name will be used as the connection string name when referenced in a dependency.</param> | ||
| /// <param name="port">The host port to bind the underlying container to.</param> | ||
| /// <returns>A reference to the <see cref="IResourceBuilder{T}"/>.</returns> | ||
| public static IResourceBuilder<ElasticsearchResource> AddElasticsearch(this IDistributedApplicationBuilder builder, [ResourceName] string name, int? port = null) | ||
| { | ||
| ArgumentNullException.ThrowIfNull(builder); | ||
| ArgumentNullException.ThrowIfNull(name); | ||
|
|
||
| var elasticsearch = new ElasticsearchResource(name); | ||
|
|
||
| string? connectionString = null; | ||
| ElasticsearchOptions? options = null; | ||
|
|
||
| builder.Eventing.Subscribe<ConnectionStringAvailableEvent>(elasticsearch, async (@event, ct) => | ||
| { | ||
| connectionString = await elasticsearch.ConnectionStringExpression.GetValueAsync(ct).ConfigureAwait(false); | ||
| if (connectionString is null) | ||
| { | ||
| throw new DistributedApplicationException($"ConnectionStringAvailableEvent was published for the '{elasticsearch.Name}' resource but the connection string was null."); | ||
| } | ||
|
|
||
| options = new ElasticsearchOptions(); | ||
| options.UseServer(connectionString); | ||
| }); | ||
|
|
||
| var healthCheckKey = $"{name}_check"; | ||
| builder.Services.AddHealthChecks() | ||
| .Add(new HealthCheckRegistration( | ||
| healthCheckKey, | ||
| sp => new ElasticsearchHealthCheck(options!), | ||
| failureStatus: default, | ||
| tags: default, | ||
| timeout: default)); | ||
|
|
||
| return builder.AddResource(elasticsearch) | ||
| .WithImage(ElasticsearchContainerImageTags.Image, ElasticsearchContainerImageTags.Tag) | ||
| .WithImageRegistry(ElasticsearchContainerImageTags.ElasticsearchRegistry) | ||
| .WithHttpEndpoint(targetPort: ElasticsearchPort, port: port, name: ElasticsearchResource.PrimaryEndpointName) | ||
| .WithEndpoint(targetPort: ElasticsearchInternalPort, name: ElasticsearchResource.InternalEndpointName) | ||
| .WithEnvironment("discovery.type", "single-node") | ||
| .WithEnvironment("xpack.security.enabled", "false") | ||
| .WithEnvironment("action.destructive_requires_name", "false") | ||
| .WithEnvironment("ES_JAVA_OPTS", "-Xms1g -Xmx1g") | ||
| .WithHealthCheck(healthCheckKey) | ||
| .PublishAsConnectionString(); | ||
| } | ||
|
|
||
| public static IResourceBuilder<ElasticsearchResource> WithKibana(this IResourceBuilder<ElasticsearchResource> builder, Action<IResourceBuilder<KibanaResource>>? configureContainer = null, string? containerName = null) | ||
| { | ||
| ArgumentNullException.ThrowIfNull(builder); | ||
|
|
||
| if (builder.ApplicationBuilder.Resources.OfType<KibanaResource>().SingleOrDefault() is { } existingKibanaResource) | ||
| { | ||
| var builderForExistingResource = builder.ApplicationBuilder.CreateResourceBuilder(existingKibanaResource); | ||
| configureContainer?.Invoke(builderForExistingResource); | ||
| return builder; | ||
| } | ||
| else | ||
| { | ||
| containerName ??= $"{builder.Resource.Name}-kibana"; | ||
|
|
||
| builder.ApplicationBuilder.Services.TryAddLifecycleHook<KibanaConfigWriterHook>(); | ||
|
|
||
| var resource = new KibanaResource(containerName); | ||
| var resourceBuilder = builder.ApplicationBuilder.AddResource(resource) | ||
| .WithImage(ElasticsearchContainerImageTags.KibanaImage, ElasticsearchContainerImageTags.Tag) | ||
| .WithImageRegistry(ElasticsearchContainerImageTags.KibanaRegistry) | ||
| .WithHttpEndpoint(targetPort: KibanaPort, name: containerName) | ||
| .WithEnvironment("xpack.security.enabled", "false") | ||
| .ExcludeFromManifest(); | ||
|
|
||
| configureContainer?.Invoke(resourceBuilder); | ||
|
|
||
| return builder; | ||
| } | ||
| } | ||
|
|
||
| public static IResourceBuilder<ElasticsearchResource> WithDataVolume(this IResourceBuilder<ElasticsearchResource> builder, string? name = null) | ||
| { | ||
| ArgumentNullException.ThrowIfNull(builder); | ||
|
|
||
| return builder.WithVolume(name ?? VolumeNameGenerator.CreateVolumeName(builder, "data"), "/usr/share/elasticsearch/data"); | ||
| } | ||
|
|
||
| public static IResourceBuilder<ElasticsearchResource> WithDataBindMount(this IResourceBuilder<ElasticsearchResource> builder, string source) | ||
| { | ||
| ArgumentNullException.ThrowIfNull(builder); | ||
| ArgumentNullException.ThrowIfNull(source); | ||
|
|
||
| return builder.WithBindMount(source, "/usr/share/elasticsearch/data"); | ||
| } | ||
| } | ||
|
|
||
| internal static class ElasticsearchContainerImageTags | ||
| { | ||
| public const string ElasticsearchRegistry = "docker.io"; | ||
| public const string Image = "exceptionless/elasticsearch"; | ||
| public const string KibanaRegistry = "docker.elastic.co"; | ||
| public const string KibanaImage = "kibana/kibana"; | ||
| public const string Tag = "8.17.0"; | ||
| } |
72 changes: 72 additions & 0 deletions
72
src/Exceptionless.AppHost/Extensions/ElasticsearchResource.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,72 @@ | ||
| namespace Aspire.Hosting; | ||
|
|
||
| /// <summary> | ||
| /// A resource that represents a Elasticsearch resource independent of the hosting model. | ||
| /// </summary> | ||
| public class ElasticsearchResource : ContainerResource, IResourceWithConnectionString | ||
| { | ||
| // this endpoint is used for all API calls over HTTP. | ||
| // This includes search and aggregations, monitoring and anything else that uses a HTTP request. | ||
| // All client libraries will use this port to talk to Elasticsearch | ||
| internal const string PrimaryEndpointName = "http"; | ||
|
|
||
| //this endpoint is a custom binary protocol used for communications between nodes in a cluster. | ||
| //For things like cluster updates, master elections, nodes joining/leaving, shard allocation | ||
| internal const string InternalEndpointName = "internal"; | ||
|
|
||
| /// <param name="name">The name of the resource.</param> | ||
| public ElasticsearchResource(string name) : base(name) | ||
| { | ||
| } | ||
|
|
||
| private EndpointReference? _primaryEndpoint; | ||
| private EndpointReference? _internalEndpoint; | ||
|
|
||
| /// <summary> | ||
| /// Gets the primary endpoint for the Elasticsearch. This endpoint is used for all API calls over HTTP. | ||
| /// </summary> | ||
| public EndpointReference PrimaryEndpoint => _primaryEndpoint ??= new(this, PrimaryEndpointName); | ||
|
|
||
| /// <summary> | ||
| /// Gets the internal endpoint for the Elasticsearch. This endpoint used for communications between nodes in a cluster | ||
| /// </summary> | ||
| public EndpointReference InternalEndpoint => _internalEndpoint ??= new(this, InternalEndpointName); | ||
|
|
||
| /// <summary> | ||
| /// Gets the connection string expression for the Elasticsearch | ||
| /// </summary> | ||
| public ReferenceExpression ConnectionString => | ||
| ReferenceExpression.Create($"http://{PrimaryEndpoint.Property(EndpointProperty.Host)}:{PrimaryEndpoint.Property(EndpointProperty.Port)}"); | ||
|
|
||
|
|
||
| /// <summary> | ||
| /// Gets the connection string expression for the Elasticsearch server for the manifest. | ||
| /// </summary> | ||
| public ReferenceExpression ConnectionStringExpression | ||
| { | ||
| get | ||
| { | ||
| if (this.TryGetLastAnnotation<ConnectionStringRedirectAnnotation>(out var connectionStringAnnotation)) | ||
| { | ||
| return connectionStringAnnotation.Resource.ConnectionStringExpression; | ||
| } | ||
|
|
||
| return ConnectionString; | ||
| } | ||
| } | ||
|
|
||
| /// <summary> | ||
| /// Gets the connection string for the Elasticsearch server. | ||
| /// </summary> | ||
| /// <param name="cancellationToken"> A <see cref="CancellationToken"/> to observe while waiting for the task to complete.</param> | ||
| /// <returns>A connection string for the Elasticsearch server in the form "http://host:port".</returns> | ||
| public ValueTask<string?> GetConnectionStringAsync(CancellationToken cancellationToken = default) | ||
| { | ||
| if (this.TryGetLastAnnotation<ConnectionStringRedirectAnnotation>(out var connectionStringAnnotation)) | ||
| { | ||
| return connectionStringAnnotation.Resource.GetConnectionStringAsync(cancellationToken); | ||
| } | ||
|
|
||
| return ConnectionString.GetValueAsync(cancellationToken); | ||
| } | ||
| } |
37 changes: 37 additions & 0 deletions
37
src/Exceptionless.AppHost/Extensions/KibanaConfigWriterHook.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,37 @@ | ||
| using System.Text; | ||
| using Aspire.Hosting.Lifecycle; | ||
| using Microsoft.Extensions.DependencyInjection; | ||
|
|
||
| namespace Aspire.Hosting; | ||
|
|
||
| internal class KibanaConfigWriterHook : IDistributedApplicationLifecycleHook | ||
| { | ||
| public async Task AfterEndpointsAllocatedAsync(DistributedApplicationModel appModel, CancellationToken cancellationToken) | ||
| { | ||
| if (appModel.Resources.OfType<KibanaResource>().SingleOrDefault() is not { } kibanaResource) | ||
| return; | ||
|
|
||
| var elasticsearchInstances = appModel.Resources.OfType<ElasticsearchResource>(); | ||
|
|
||
| if (!elasticsearchInstances.Any()) | ||
| return; | ||
|
|
||
| var hostsVariableBuilder = new StringBuilder(); | ||
|
|
||
| foreach (var elasticsearchInstance in elasticsearchInstances) | ||
| { | ||
| if (elasticsearchInstance.PrimaryEndpoint.IsAllocated) | ||
| { | ||
| var connectionString = await elasticsearchInstance.GetConnectionStringAsync(); | ||
| if (hostsVariableBuilder.Length > 0) | ||
| hostsVariableBuilder.Append(","); | ||
| hostsVariableBuilder.Append(elasticsearchInstance.PrimaryEndpoint.Scheme).Append("://").Append(elasticsearchInstance.PrimaryEndpoint.ContainerHost).Append(":").Append(elasticsearchInstance.PrimaryEndpoint.Port); | ||
| } | ||
| } | ||
|
|
||
| kibanaResource.Annotations.Add(new EnvironmentCallbackAnnotation(context => | ||
| { | ||
| context.EnvironmentVariables.Add("ELASTICSEARCH_HOSTS", hostsVariableBuilder.ToString()); | ||
| })); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,9 @@ | ||
| namespace Aspire.Hosting; | ||
|
|
||
| /// <summary> | ||
| /// A resource that represents a Kibana container. | ||
| /// </summary> | ||
| /// <param name="name">The name of the resource.</param> | ||
| public class KibanaResource(string name) : ContainerResource(name) | ||
| { | ||
| } |
Oops, something went wrong.