re-enable e2e workflow (#39)

* re-enable e2e workflow

* dump log file to xunit test output if test has failures

* remodel nullability on distributed fixture

* Update to latest nullean xunit

* include log and log to console for good measure

* console write log

* lenient log glob

* fix glob :(

* fix E2E authorization header

* update wait timeout

* run dotnet format again

* never dump playwright trace bootstrap on CI

* artifact trace output

* unpack last screenshot and attach that to PR

* update .NET formatting

* wait longer for started confirmation on distributed application launc

.github/workflows/e2e.yml

@@ -31,10 +31,9 @@ env:
 #       update e2e-docs.yml
 jobs:
   test:
-    if: false()
-    #if: |
-    # github.event_name != 'pull_request' || 
-    # (github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == false)
+    if: |
+      github.event_name != 'pull_request' || 
+      (github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == false)
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
@@ -70,4 +69,11 @@ jobs:
       - run: ./build.sh test --test-suite=e2e
         env: 
           E2E__ENDPOINT: "${{env.ELASTIC_APM_SERVER_URL}}"
-          E2E__AUTHORIZATION: "Authentication=ApiKey ${{env.ELASTIC_APM_API_KEY}}"
+          E2E__AUTHORIZATION: "Authorization=ApiKey ${{env.ELASTIC_APM_API_KEY}}"
+
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: playwright-traces
+          path: .artifacts/playwright-traces/*-screenshot.jpeg
+          retention-days: 1

.gitignore

@@ -11,6 +11,7 @@
 *.user
 *.userosscache
 *.sln.docstates
+.DS_Store
 
 # User-specific files (MonoDevelop/Xamarin Studio)
 *.userprefs

tests/Elastic.OpenTelemetry.EndToEndTests/DistributedFixture/ApmUIBrowserContext.cs

@@ -2,6 +2,8 @@
 // Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
 // See the LICENSE file in the project root for more information
 
+using System.IO.Compression;
+using System.Runtime.CompilerServices;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Playwright;
 using Xunit;
@@ -29,14 +31,16 @@ public ApmUIBrowserContext(IConfigurationRoot configuration, string serviceName)
 	public IBrowser Browser { get; private set; } = null!;
 	public IPlaywright HeadlessTester { get; private set; } = null!;
 
+	private const string BootstrapTraceName = "test_bootstrap";
+
 	public async Task InitializeAsync()
 	{
 		var username = _configuration["E2E:BrowserEmail"]?.Trim() ?? string.Empty;
 		var password = _configuration["E2E:BrowserPassword"]?.Trim() ?? string.Empty;
 		Program.Main(["install", "chromium"]);
 		HeadlessTester = await Playwright.CreateAsync();
 		Browser = await HeadlessTester.Chromium.LaunchAsync();
-		var page = await OpenApmLandingPage("test_bootstrap");
+		var page = await OpenApmLandingPage(BootstrapTraceName);
 		try
 		{
 			await page.GetByRole(AriaRole.Textbox, new() { Name = "email" }).FillAsync(username);
@@ -47,11 +51,11 @@ public async Task InitializeAsync()
 
 			StorageState = await page.Context.StorageStateAsync();
 
-			await StopTrace(page);
+			await StopTrace(page, success: true);
 		}
 		catch (Exception e)
 		{
-			await StopTrace(page, "test_bootstrap");
+			await StopTrace(page, success: false);
 			Console.WriteLine(e);
 			throw;
 		}
@@ -68,7 +72,7 @@ await page.Context.Tracing.StartAsync(new()
 			Title = testName,
 			Screenshots = true,
 			Snapshots = true,
-			Sources = true
+			Sources = false
 		});
 
 		return page;
@@ -84,20 +88,21 @@ public async Task<IPage> OpenApmLandingPage(string testName)
 
 	public async Task WaitForServiceOnOverview(IPage page)
 	{
-		page.SetDefaultTimeout((float)TimeSpan.FromSeconds(30).TotalMilliseconds);
 
-		var servicesHeader = page.GetByRole(AriaRole.Heading, new() { Name = "Services" });
-		await servicesHeader.WaitForAsync(new() { State = WaitForSelectorState.Visible });
+		var timeout = (float)TimeSpan.FromSeconds(30).TotalMilliseconds;
 
-		page.SetDefaultTimeout((float)TimeSpan.FromSeconds(10).TotalMilliseconds);
+		var servicesHeader = page.GetByRole(AriaRole.Heading, new() { Name = "Services" });
+		await servicesHeader.WaitForAsync(new() { State = WaitForSelectorState.Visible, Timeout = timeout });
 
 		Exception? observed = null;
+
+		var refreshTimeout = (float)TimeSpan.FromSeconds(5).TotalMilliseconds;
 		for (var i = 0; i < 10; i++)
 		{
 			try
 			{
 				var serviceLink = page.GetByRole(AriaRole.Link, new() { Name = _serviceName });
-				await serviceLink.WaitForAsync(new() { State = WaitForSelectorState.Visible });
+				await serviceLink.WaitForAsync(new() { State = WaitForSelectorState.Visible, Timeout = refreshTimeout });
 				observed = null;
 				break;
 			}
@@ -108,30 +113,31 @@ public async Task WaitForServiceOnOverview(IPage page)
 			}
 			finally
 			{
-				page.SetDefaultTimeout((float)TimeSpan.FromSeconds(5).TotalMilliseconds);
+				page.SetDefaultTimeout(refreshTimeout);
 			}
 		}
 		if (observed != null)
 			throw observed; //TODO proper rethrow with stack
 
 	}
 
-	public async Task StopTrace(IPage page, string? testName = null)
+	private int _unnamedTests;
+	public async Task StopTrace(IPage page, bool success, [CallerMemberName] string? testName = null)
 	{
-
-		if (string.IsNullOrWhiteSpace(testName))
+		testName ??= $"unknown_test_{_unnamedTests++}";
+		//only dump trace zip of test name is provided.
+		if (success)
 			await page.Context.Tracing.StopAsync(new());
 		else
 		{
 			var root = DotNetRunApplication.GetSolutionRoot();
-			await page.Context.Tracing.StopAsync(new()
-			{
-				Path = Path.Combine(
-					Path.Combine(root.FullName, ".artifacts"),
-					"playwright-traces",
-					$"{testName}.zip"
-				)
-			});
+			var zip = Path.Combine(root.FullName, ".artifacts", "playwright-traces", $"{testName}.zip");
+			await page.Context.Tracing.StopAsync(new() { Path = zip });
+
+			using var archive = ZipFile.OpenRead(zip);
+			var entries = archive.Entries.Where(e => e.FullName.StartsWith("resources") && e.FullName.EndsWith(".jpeg")).ToList();
+			var lastScreenshot = entries.MaxBy(e => e.LastWriteTime);
+			lastScreenshot?.ExtractToFile(Path.Combine(root.FullName, ".artifacts", "playwright-traces", $"{testName}-screenshot.jpeg"));
 		}
 		await page.CloseAsync();
 	}

tests/Elastic.OpenTelemetry.EndToEndTests/DistributedFixture/DistributedApplicationFixture.cs

@@ -6,7 +6,9 @@
 using System.Security.Cryptography;
 using System.Text;
 using Microsoft.Extensions.Configuration;
+using Microsoft.VisualStudio.TestPlatform.ObjectModel;
 using Nullean.Xunit.Partitions.Sdk;
+using Xunit.Sdk;
 
 namespace Elastic.OpenTelemetry.EndToEndTests.DistributedFixture;
 
@@ -16,13 +18,28 @@ public class DistributedApplicationFixture : IPartitionLifetime
 
 	public string ServiceName { get; } = $"dotnet-e2e-{ShaForCurrentTicks()}";
 
-	public bool Started => AspNetApplication.ProcessId.HasValue;
+	public bool Started => AspNetApplication?.ProcessId.HasValue ?? false;
+
+	private readonly List<string> _output = new();
 
 	public int? MaxConcurrency => null;
 
-	public ApmUIBrowserContext ApmUI { get; private set; } = null!;
+	private ApmUIBrowserContext? _apmUI;
+	public ApmUIBrowserContext ApmUI
+	{
+		get => _apmUI ??
+			throw new NullReferenceException($"{nameof(DistributedApplicationFixture)} no yet initialized");
+		private set => _apmUI = value;
+	}
+
+	private AspNetCoreExampleApplication? _aspNetApplication;
 
-	public AspNetCoreExampleApplication AspNetApplication { get; private set; } = null!;
+	public AspNetCoreExampleApplication AspNetApplication
+	{
+		get => _aspNetApplication
+			?? throw new NullReferenceException($"{nameof(DistributedApplicationFixture)} no yet initialized");
+		private set => _aspNetApplication = value;
+	}
 
 	private static string ShaForCurrentTicks()
 	{
@@ -34,10 +51,31 @@ private static string ShaForCurrentTicks()
 			.Substring(0, 12);
 	}
 
+	public string FailureTestOutput()
+	{
+		var logLines = new List<string>();
+		if (_aspNetApplication?.ProcessId.HasValue ?? false)
+			AspNetApplication.IterateOverLog(s =>
+			{
+				Console.WriteLine(s);
+				logLines.Add(s);
+			});
+
+		var messages = string.Join(Environment.NewLine, _output.Concat(logLines));
+		return messages;
+
+	}
+
 	public async Task DisposeAsync()
 	{
-		AspNetApplication.Dispose();
-		await ApmUI.DisposeAsync();
+		_aspNetApplication?.Dispose();
+		await (_apmUI?.DisposeAsync() ?? Task.CompletedTask);
+	}
+
+	private void Log(string message)
+	{
+		Console.WriteLine(message);
+		_output.Add(message);
 	}
 
 	public async Task InitializeAsync()
@@ -47,18 +85,32 @@ public async Task InitializeAsync()
 			.AddUserSecrets<DotNetRunApplication>()
 			.Build();
 
+		Log("Created configuration");
+
 		AspNetApplication = new AspNetCoreExampleApplication(ServiceName, configuration);
+
+		Log("Started ASP.NET application");
+
 		ApmUI = new ApmUIBrowserContext(configuration, ServiceName);
 
+		Log("Started UI Browser context");
+
 		foreach (var trafficSimulator in _trafficSimulators)
 			await trafficSimulator.Start(this);
 
+		Log("Simulated traffic");
+
 		// TODO query OTEL_BSP_SCHEDULE_DELAY?
 		await Task.Delay(5000);
 
+		Log("Waited for OTEL_BSP_SCHEDULE_DELAY");
+
 		// Stateless refresh
 		//https://github.com/elastic/elasticsearch/blob/main/server/src/main/java/org/elasticsearch/index/IndexSettings.java#L286
 		await Task.Delay(TimeSpan.FromSeconds(15));
+
+		Log("Waited for Stateless refresh");
+
 		await ApmUI.InitializeAsync();
 	}
 

tests/Elastic.OpenTelemetry.EndToEndTests/DistributedFixture/DotNetRunApplication.cs

@@ -13,21 +13,25 @@ public abstract class DotNetRunApplication
 {
 	private static readonly DirectoryInfo CurrentDirectory = new FileInfo(Assembly.GetExecutingAssembly().Location).Directory!;
 	private static readonly Regex ProcessIdMatch = new(@"^\s*Process Id (?<processid>\d+)");
+
+	public static readonly DirectoryInfo Root = GetSolutionRoot();
+	public static readonly DirectoryInfo LogDirectory = new(Path.Combine(Root.FullName, ".artifacts", "tests"));
+
 	private readonly LongRunningApplicationSubscription _app;
 	private readonly string _applicationName;
 	private readonly string _authorization;
 	private readonly string _endpoint;
 	private readonly string _serviceName;
 
-	public DotNetRunApplication(string serviceName, IConfiguration configuration, string applicationName)
+	protected DotNetRunApplication(string serviceName, IConfiguration configuration, string applicationName)
 	{
 		_serviceName = serviceName;
 		_applicationName = applicationName;
 		_endpoint = configuration["E2E:Endpoint"]?.Trim() ?? string.Empty;
 		_authorization = configuration["E2E:Authorization"]?.Trim() ?? string.Empty;
 
 		var args = CreateStartArgs();
-		_app = Proc.StartLongRunning(args, TimeSpan.FromSeconds(10));
+		_app = Proc.StartLongRunning(args, TimeSpan.FromSeconds(30));
 	}
 
 	public int? ProcessId { get; private set; }
@@ -48,8 +52,7 @@ public static DirectoryInfo GetSolutionRoot()
 
 	private LongRunningArguments CreateStartArgs()
 	{
-		var root = GetSolutionRoot();
-		var project = Path.Combine(root.FullName, "examples", _applicationName);
+		var project = Path.Combine(Root.FullName, "examples", _applicationName);
 
 		var arguments = new[] { "run", "--project", project };
 		var applicationArguments = GetArguments();
@@ -58,6 +61,7 @@ private LongRunningArguments CreateStartArgs()
 
 		return new("dotnet", arguments)
 		{
+
 			Environment = new Dictionary<string, string>
 			{
 				{ "OTEL_EXPORTER_OTLP_ENDPOINT", _endpoint },
@@ -67,6 +71,10 @@ private LongRunningArguments CreateStartArgs()
 				{ "OTEL_BSP_SCHEDULE_DELAY", "1000" },
 				{ "OTEL_BSP_MAX_EXPORT_BATCH_SIZE", "5" },
 				{ "OTEL_RESOURCE_ATTRIBUTES", $"service.name={_serviceName},service.version=1.0,1,deployment.environment=e2e" },
+
+				{ "ELASTIC_OTEL_ENABLE_FILE_LOGGING", "1" },
+				{ "ELASTIC_OTEL_LOG_DIRECTORY", LogDirectory.FullName },
+				{ "ELASTIC_OTEL_LOG_LEVEL", "INFO" },
 			},
 			StartedConfirmationHandler = l =>
 			{
@@ -80,6 +88,28 @@ private LongRunningArguments CreateStartArgs()
 				return l.Line.StartsWith("      Application started.");
 			}
 		};
+
+
+	}
+
+	public void IterateOverLog(Action<string> write)
+	{
+		var logFile = DotNetRunApplication.LogDirectory
+			 //TODO get last of this app specifically
+			 //.GetFiles($"{_app.Process.Binary}_*.log")
+			 .GetFiles($"*.log")
+			 .MaxBy(f => f.CreationTimeUtc);
+
+		if (logFile == null)
+			write($"Could not locate log files in {DotNetRunApplication.LogDirectory}");
+		else
+		{
+			write($"Contents of: {logFile.FullName}");
+			using var sr = logFile.OpenText();
+			var s = string.Empty;
+			while ((s = sr.ReadLine()) != null)
+				write(s);
+		}
 	}
 
 	public virtual void Dispose()

tests/Elastic.OpenTelemetry.EndToEndTests/Elastic.OpenTelemetry.EndToEndTests.csproj

@@ -12,7 +12,7 @@
 
   <ItemGroup>
     <PackageReference Include="Microsoft.Playwright" Version="1.41.1"/>
-    <PackageReference Include="Nullean.Xunit.Partitions" Version="0.4.3" />
+    <PackageReference Include="Nullean.Xunit.Partitions" Version="0.5.0" />
     <PackageReference Include="Proc" Version="0.8.1"/>
     <PackageReference Include="Microsoft.Extensions.Configuration" Version="8.0.0"/>
     <PackageReference Include="Microsoft.Extensions.Configuration.UserSecrets" Version="8.0.0"/>

tests/Elastic.OpenTelemetry.EndToEndTests/ServiceTests.cs

@@ -25,15 +25,26 @@ public class EndToEndTests(ITestOutputHelper output, DistributedApplicationFixtu
 	[Fact]
 	public async Task LatencyShowsAGraph()
 	{
+		var timeout = (float)TimeSpan.FromSeconds(30).TotalMilliseconds;
+
 		// click on service in service overview page.
-		_page.SetDefaultTimeout((float)TimeSpan.FromSeconds(30).TotalMilliseconds);
 		var uri = new Uri(fixture.ApmUI.KibanaAppUri, $"/app/apm/services/{fixture.ServiceName}/overview").ToString();
-		await _page.GotoAsync(uri);
-		await Expect(_page.GetByRole(AriaRole.Heading, new() { Name = "Latency", Exact = true })).ToBeVisibleAsync();
+		await _page.GotoAsync(uri, new() { Timeout = timeout });
+		await Expect(_page.GetByRole(AriaRole.Heading, new() { Name = "Latency", Exact = true }))
+			.ToBeVisibleAsync(new() { Timeout = timeout });
 	}
 
 
 	public async Task InitializeAsync() => _page = await fixture.ApmUI.NewProfiledPage(_testName);
 
-	public async Task DisposeAsync() => await fixture.ApmUI.StopTrace(_page, PartitionContext.TestException == null ? null : _testName);
+	public async Task DisposeAsync()
+	{
+		var success = PartitionContext.TestException == null;
+		await fixture.ApmUI.StopTrace(_page, success, _testName);
+
+		if (!success)
+			return;
+
+		fixture.AspNetApplication.IterateOverLog(Output.WriteLine);
+	}
 }