From f9c1136f692a8b25702a4c1203bc3fe19b207140 Mon Sep 17 00:00:00 2001 From: Martijn Laarman Date: Fri, 8 Mar 2024 09:04:12 +0100 Subject: [PATCH] re-enable e2e workflow (#39) * re-enable e2e workflow * dump log file to xunit test output if test has failures * remodel nullability on distributed fixture * Update to latest nullean xunit * include log and log to console for good measure * console write log * lenient log glob * fix glob :( * fix E2E authorization header * update wait timeout * run dotnet format again * never dump playwright trace bootstrap on CI * artifact trace output * unpack last screenshot and attach that to PR * update .NET formatting * wait longer for started confirmation on distributed application launc --- .github/workflows/e2e.yml | 16 +++-- .gitignore | 1 + .../DistributedFixture/ApmUIBrowserContext.cs | 48 +++++++------- .../DistributedApplicationFixture.cs | 62 +++++++++++++++++-- .../DotNetRunApplication.cs | 38 ++++++++++-- ...Elastic.OpenTelemetry.EndToEndTests.csproj | 2 +- .../ServiceTests.cs | 19 ++++-- .../LoggingTests.cs | 1 - 8 files changed, 146 insertions(+), 41 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index fa2cc99..c96ec36 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -31,10 +31,9 @@ env: # update e2e-docs.yml jobs: test: - if: false() - #if: | - # github.event_name != 'pull_request' || - # (github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == false) + if: | + github.event_name != 'pull_request' || + (github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == false) runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -70,4 +69,11 @@ jobs: - run: ./build.sh test --test-suite=e2e env: E2E__ENDPOINT: "${{env.ELASTIC_APM_SERVER_URL}}" - E2E__AUTHORIZATION: "Authentication=ApiKey ${{env.ELASTIC_APM_API_KEY}}" \ No newline at end of file + E2E__AUTHORIZATION: "Authorization=ApiKey ${{env.ELASTIC_APM_API_KEY}}" + + - uses: actions/upload-artifact@v4 + if: always() + with: + name: playwright-traces + path: .artifacts/playwright-traces/*-screenshot.jpeg + retention-days: 1 \ No newline at end of file diff --git a/.gitignore b/.gitignore index 6e81bda..c87b8d0 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ *.user *.userosscache *.sln.docstates +.DS_Store # User-specific files (MonoDevelop/Xamarin Studio) *.userprefs diff --git a/tests/Elastic.OpenTelemetry.EndToEndTests/DistributedFixture/ApmUIBrowserContext.cs b/tests/Elastic.OpenTelemetry.EndToEndTests/DistributedFixture/ApmUIBrowserContext.cs index 9db31d3..a66bb01 100644 --- a/tests/Elastic.OpenTelemetry.EndToEndTests/DistributedFixture/ApmUIBrowserContext.cs +++ b/tests/Elastic.OpenTelemetry.EndToEndTests/DistributedFixture/ApmUIBrowserContext.cs @@ -2,6 +2,8 @@ // Elasticsearch B.V licenses this file to you under the Apache 2.0 License. // See the LICENSE file in the project root for more information +using System.IO.Compression; +using System.Runtime.CompilerServices; using Microsoft.Extensions.Configuration; using Microsoft.Playwright; using Xunit; @@ -29,6 +31,8 @@ public ApmUIBrowserContext(IConfigurationRoot configuration, string serviceName) public IBrowser Browser { get; private set; } = null!; public IPlaywright HeadlessTester { get; private set; } = null!; + private const string BootstrapTraceName = "test_bootstrap"; + public async Task InitializeAsync() { var username = _configuration["E2E:BrowserEmail"]?.Trim() ?? string.Empty; @@ -36,7 +40,7 @@ public async Task InitializeAsync() Program.Main(["install", "chromium"]); HeadlessTester = await Playwright.CreateAsync(); Browser = await HeadlessTester.Chromium.LaunchAsync(); - var page = await OpenApmLandingPage("test_bootstrap"); + var page = await OpenApmLandingPage(BootstrapTraceName); try { await page.GetByRole(AriaRole.Textbox, new() { Name = "email" }).FillAsync(username); @@ -47,11 +51,11 @@ public async Task InitializeAsync() StorageState = await page.Context.StorageStateAsync(); - await StopTrace(page); + await StopTrace(page, success: true); } catch (Exception e) { - await StopTrace(page, "test_bootstrap"); + await StopTrace(page, success: false); Console.WriteLine(e); throw; } @@ -68,7 +72,7 @@ await page.Context.Tracing.StartAsync(new() Title = testName, Screenshots = true, Snapshots = true, - Sources = true + Sources = false }); return page; @@ -84,20 +88,21 @@ public async Task OpenApmLandingPage(string testName) public async Task WaitForServiceOnOverview(IPage page) { - page.SetDefaultTimeout((float)TimeSpan.FromSeconds(30).TotalMilliseconds); - var servicesHeader = page.GetByRole(AriaRole.Heading, new() { Name = "Services" }); - await servicesHeader.WaitForAsync(new() { State = WaitForSelectorState.Visible }); + var timeout = (float)TimeSpan.FromSeconds(30).TotalMilliseconds; - page.SetDefaultTimeout((float)TimeSpan.FromSeconds(10).TotalMilliseconds); + var servicesHeader = page.GetByRole(AriaRole.Heading, new() { Name = "Services" }); + await servicesHeader.WaitForAsync(new() { State = WaitForSelectorState.Visible, Timeout = timeout }); Exception? observed = null; + + var refreshTimeout = (float)TimeSpan.FromSeconds(5).TotalMilliseconds; for (var i = 0; i < 10; i++) { try { var serviceLink = page.GetByRole(AriaRole.Link, new() { Name = _serviceName }); - await serviceLink.WaitForAsync(new() { State = WaitForSelectorState.Visible }); + await serviceLink.WaitForAsync(new() { State = WaitForSelectorState.Visible, Timeout = refreshTimeout }); observed = null; break; } @@ -108,7 +113,7 @@ public async Task WaitForServiceOnOverview(IPage page) } finally { - page.SetDefaultTimeout((float)TimeSpan.FromSeconds(5).TotalMilliseconds); + page.SetDefaultTimeout(refreshTimeout); } } if (observed != null) @@ -116,22 +121,23 @@ public async Task WaitForServiceOnOverview(IPage page) } - public async Task StopTrace(IPage page, string? testName = null) + private int _unnamedTests; + public async Task StopTrace(IPage page, bool success, [CallerMemberName] string? testName = null) { - - if (string.IsNullOrWhiteSpace(testName)) + testName ??= $"unknown_test_{_unnamedTests++}"; + //only dump trace zip of test name is provided. + if (success) await page.Context.Tracing.StopAsync(new()); else { var root = DotNetRunApplication.GetSolutionRoot(); - await page.Context.Tracing.StopAsync(new() - { - Path = Path.Combine( - Path.Combine(root.FullName, ".artifacts"), - "playwright-traces", - $"{testName}.zip" - ) - }); + var zip = Path.Combine(root.FullName, ".artifacts", "playwright-traces", $"{testName}.zip"); + await page.Context.Tracing.StopAsync(new() { Path = zip }); + + using var archive = ZipFile.OpenRead(zip); + var entries = archive.Entries.Where(e => e.FullName.StartsWith("resources") && e.FullName.EndsWith(".jpeg")).ToList(); + var lastScreenshot = entries.MaxBy(e => e.LastWriteTime); + lastScreenshot?.ExtractToFile(Path.Combine(root.FullName, ".artifacts", "playwright-traces", $"{testName}-screenshot.jpeg")); } await page.CloseAsync(); } diff --git a/tests/Elastic.OpenTelemetry.EndToEndTests/DistributedFixture/DistributedApplicationFixture.cs b/tests/Elastic.OpenTelemetry.EndToEndTests/DistributedFixture/DistributedApplicationFixture.cs index c17af1b..352ee4e 100644 --- a/tests/Elastic.OpenTelemetry.EndToEndTests/DistributedFixture/DistributedApplicationFixture.cs +++ b/tests/Elastic.OpenTelemetry.EndToEndTests/DistributedFixture/DistributedApplicationFixture.cs @@ -6,7 +6,9 @@ using System.Security.Cryptography; using System.Text; using Microsoft.Extensions.Configuration; +using Microsoft.VisualStudio.TestPlatform.ObjectModel; using Nullean.Xunit.Partitions.Sdk; +using Xunit.Sdk; namespace Elastic.OpenTelemetry.EndToEndTests.DistributedFixture; @@ -16,13 +18,28 @@ public class DistributedApplicationFixture : IPartitionLifetime public string ServiceName { get; } = $"dotnet-e2e-{ShaForCurrentTicks()}"; - public bool Started => AspNetApplication.ProcessId.HasValue; + public bool Started => AspNetApplication?.ProcessId.HasValue ?? false; + + private readonly List _output = new(); public int? MaxConcurrency => null; - public ApmUIBrowserContext ApmUI { get; private set; } = null!; + private ApmUIBrowserContext? _apmUI; + public ApmUIBrowserContext ApmUI + { + get => _apmUI ?? + throw new NullReferenceException($"{nameof(DistributedApplicationFixture)} no yet initialized"); + private set => _apmUI = value; + } + + private AspNetCoreExampleApplication? _aspNetApplication; - public AspNetCoreExampleApplication AspNetApplication { get; private set; } = null!; + public AspNetCoreExampleApplication AspNetApplication + { + get => _aspNetApplication + ?? throw new NullReferenceException($"{nameof(DistributedApplicationFixture)} no yet initialized"); + private set => _aspNetApplication = value; + } private static string ShaForCurrentTicks() { @@ -34,10 +51,31 @@ private static string ShaForCurrentTicks() .Substring(0, 12); } + public string FailureTestOutput() + { + var logLines = new List(); + if (_aspNetApplication?.ProcessId.HasValue ?? false) + AspNetApplication.IterateOverLog(s => + { + Console.WriteLine(s); + logLines.Add(s); + }); + + var messages = string.Join(Environment.NewLine, _output.Concat(logLines)); + return messages; + + } + public async Task DisposeAsync() { - AspNetApplication.Dispose(); - await ApmUI.DisposeAsync(); + _aspNetApplication?.Dispose(); + await (_apmUI?.DisposeAsync() ?? Task.CompletedTask); + } + + private void Log(string message) + { + Console.WriteLine(message); + _output.Add(message); } public async Task InitializeAsync() @@ -47,18 +85,32 @@ public async Task InitializeAsync() .AddUserSecrets() .Build(); + Log("Created configuration"); + AspNetApplication = new AspNetCoreExampleApplication(ServiceName, configuration); + + Log("Started ASP.NET application"); + ApmUI = new ApmUIBrowserContext(configuration, ServiceName); + Log("Started UI Browser context"); + foreach (var trafficSimulator in _trafficSimulators) await trafficSimulator.Start(this); + Log("Simulated traffic"); + // TODO query OTEL_BSP_SCHEDULE_DELAY? await Task.Delay(5000); + Log("Waited for OTEL_BSP_SCHEDULE_DELAY"); + // Stateless refresh //https://github.com/elastic/elasticsearch/blob/main/server/src/main/java/org/elasticsearch/index/IndexSettings.java#L286 await Task.Delay(TimeSpan.FromSeconds(15)); + + Log("Waited for Stateless refresh"); + await ApmUI.InitializeAsync(); } diff --git a/tests/Elastic.OpenTelemetry.EndToEndTests/DistributedFixture/DotNetRunApplication.cs b/tests/Elastic.OpenTelemetry.EndToEndTests/DistributedFixture/DotNetRunApplication.cs index cfd7a89..c9d4aef 100644 --- a/tests/Elastic.OpenTelemetry.EndToEndTests/DistributedFixture/DotNetRunApplication.cs +++ b/tests/Elastic.OpenTelemetry.EndToEndTests/DistributedFixture/DotNetRunApplication.cs @@ -13,13 +13,17 @@ public abstract class DotNetRunApplication { private static readonly DirectoryInfo CurrentDirectory = new FileInfo(Assembly.GetExecutingAssembly().Location).Directory!; private static readonly Regex ProcessIdMatch = new(@"^\s*Process Id (?\d+)"); + + public static readonly DirectoryInfo Root = GetSolutionRoot(); + public static readonly DirectoryInfo LogDirectory = new(Path.Combine(Root.FullName, ".artifacts", "tests")); + private readonly LongRunningApplicationSubscription _app; private readonly string _applicationName; private readonly string _authorization; private readonly string _endpoint; private readonly string _serviceName; - public DotNetRunApplication(string serviceName, IConfiguration configuration, string applicationName) + protected DotNetRunApplication(string serviceName, IConfiguration configuration, string applicationName) { _serviceName = serviceName; _applicationName = applicationName; @@ -27,7 +31,7 @@ public DotNetRunApplication(string serviceName, IConfiguration configuration, st _authorization = configuration["E2E:Authorization"]?.Trim() ?? string.Empty; var args = CreateStartArgs(); - _app = Proc.StartLongRunning(args, TimeSpan.FromSeconds(10)); + _app = Proc.StartLongRunning(args, TimeSpan.FromSeconds(30)); } public int? ProcessId { get; private set; } @@ -48,8 +52,7 @@ public static DirectoryInfo GetSolutionRoot() private LongRunningArguments CreateStartArgs() { - var root = GetSolutionRoot(); - var project = Path.Combine(root.FullName, "examples", _applicationName); + var project = Path.Combine(Root.FullName, "examples", _applicationName); var arguments = new[] { "run", "--project", project }; var applicationArguments = GetArguments(); @@ -58,6 +61,7 @@ private LongRunningArguments CreateStartArgs() return new("dotnet", arguments) { + Environment = new Dictionary { { "OTEL_EXPORTER_OTLP_ENDPOINT", _endpoint }, @@ -67,6 +71,10 @@ private LongRunningArguments CreateStartArgs() { "OTEL_BSP_SCHEDULE_DELAY", "1000" }, { "OTEL_BSP_MAX_EXPORT_BATCH_SIZE", "5" }, { "OTEL_RESOURCE_ATTRIBUTES", $"service.name={_serviceName},service.version=1.0,1,deployment.environment=e2e" }, + + { "ELASTIC_OTEL_ENABLE_FILE_LOGGING", "1" }, + { "ELASTIC_OTEL_LOG_DIRECTORY", LogDirectory.FullName }, + { "ELASTIC_OTEL_LOG_LEVEL", "INFO" }, }, StartedConfirmationHandler = l => { @@ -80,6 +88,28 @@ private LongRunningArguments CreateStartArgs() return l.Line.StartsWith(" Application started."); } }; + + + } + + public void IterateOverLog(Action write) + { + var logFile = DotNetRunApplication.LogDirectory + //TODO get last of this app specifically + //.GetFiles($"{_app.Process.Binary}_*.log") + .GetFiles($"*.log") + .MaxBy(f => f.CreationTimeUtc); + + if (logFile == null) + write($"Could not locate log files in {DotNetRunApplication.LogDirectory}"); + else + { + write($"Contents of: {logFile.FullName}"); + using var sr = logFile.OpenText(); + var s = string.Empty; + while ((s = sr.ReadLine()) != null) + write(s); + } } public virtual void Dispose() diff --git a/tests/Elastic.OpenTelemetry.EndToEndTests/Elastic.OpenTelemetry.EndToEndTests.csproj b/tests/Elastic.OpenTelemetry.EndToEndTests/Elastic.OpenTelemetry.EndToEndTests.csproj index 41dfa91..644d8c1 100644 --- a/tests/Elastic.OpenTelemetry.EndToEndTests/Elastic.OpenTelemetry.EndToEndTests.csproj +++ b/tests/Elastic.OpenTelemetry.EndToEndTests/Elastic.OpenTelemetry.EndToEndTests.csproj @@ -12,7 +12,7 @@ - + diff --git a/tests/Elastic.OpenTelemetry.EndToEndTests/ServiceTests.cs b/tests/Elastic.OpenTelemetry.EndToEndTests/ServiceTests.cs index 195ad63..8725a6f 100644 --- a/tests/Elastic.OpenTelemetry.EndToEndTests/ServiceTests.cs +++ b/tests/Elastic.OpenTelemetry.EndToEndTests/ServiceTests.cs @@ -25,15 +25,26 @@ public class EndToEndTests(ITestOutputHelper output, DistributedApplicationFixtu [Fact] public async Task LatencyShowsAGraph() { + var timeout = (float)TimeSpan.FromSeconds(30).TotalMilliseconds; + // click on service in service overview page. - _page.SetDefaultTimeout((float)TimeSpan.FromSeconds(30).TotalMilliseconds); var uri = new Uri(fixture.ApmUI.KibanaAppUri, $"/app/apm/services/{fixture.ServiceName}/overview").ToString(); - await _page.GotoAsync(uri); - await Expect(_page.GetByRole(AriaRole.Heading, new() { Name = "Latency", Exact = true })).ToBeVisibleAsync(); + await _page.GotoAsync(uri, new() { Timeout = timeout }); + await Expect(_page.GetByRole(AriaRole.Heading, new() { Name = "Latency", Exact = true })) + .ToBeVisibleAsync(new() { Timeout = timeout }); } public async Task InitializeAsync() => _page = await fixture.ApmUI.NewProfiledPage(_testName); - public async Task DisposeAsync() => await fixture.ApmUI.StopTrace(_page, PartitionContext.TestException == null ? null : _testName); + public async Task DisposeAsync() + { + var success = PartitionContext.TestException == null; + await fixture.ApmUI.StopTrace(_page, success, _testName); + + if (!success) + return; + + fixture.AspNetApplication.IterateOverLog(Output.WriteLine); + } } diff --git a/tests/Elastic.OpenTelemetry.Tests/LoggingTests.cs b/tests/Elastic.OpenTelemetry.Tests/LoggingTests.cs index e3748a0..5033eba 100644 --- a/tests/Elastic.OpenTelemetry.Tests/LoggingTests.cs +++ b/tests/Elastic.OpenTelemetry.Tests/LoggingTests.cs @@ -2,7 +2,6 @@ // Elasticsearch B.V licenses this file to you under the Apache 2.0 License. // See the LICENSE file in the project root for more information -using System.Diagnostics; using OpenTelemetry; using Xunit.Abstractions;