From 2f557816f4024edc72b2e62254a26f11043ec599 Mon Sep 17 00:00:00 2001 From: Dave Tryon <45672944+DaveTryon@users.noreply.github.com> Date: Tue, 9 Jul 2024 12:10:00 -0700 Subject: [PATCH] Add simple integration tests (#606) * Add simple integration tests * Make app name OS-aware * Remove unneeded using statements * Tweak non-Windows platforms * Missed 1 test case * Try calling chmod u+x on test app * Skip all but 1 test on non-Windows * PR feedback * Create a separate folder for each test, as intended * Use TestRunDirectory instead of TestResultsDirectory --- Microsoft.Sbom.sln | 6 + .../IntegrationTests.cs | 234 ++++++++++++++++++ .../Microsoft.Sbom.Tool.Tests.csproj | 18 ++ 3 files changed, 258 insertions(+) create mode 100644 test/Microsoft.Sbom.Tool.Tests/IntegrationTests.cs create mode 100644 test/Microsoft.Sbom.Tool.Tests/Microsoft.Sbom.Tool.Tests.csproj diff --git a/Microsoft.Sbom.sln b/Microsoft.Sbom.sln index bf99d14f..199c973b 100644 --- a/Microsoft.Sbom.sln +++ b/Microsoft.Sbom.sln @@ -49,6 +49,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Sbom.Extensions.D EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Sbom.Extensions.DependencyInjection.Tests", "test\Microsoft.Sbom.Extensions.DependencyInjection.Tests\Microsoft.Sbom.Extensions.DependencyInjection.Tests.csproj", "{EE4E2E03-7B4C-46E5-B9D2-89E84A18D787}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Sbom.Tool.Tests", "test\Microsoft.Sbom.Tool.Tests\Microsoft.Sbom.Tool.Tests.csproj", "{FC5A9799-7C44-4BFA-BA22-55DCAF1A1B9F}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -107,6 +109,10 @@ Global {EE4E2E03-7B4C-46E5-B9D2-89E84A18D787}.Debug|Any CPU.Build.0 = Debug|Any CPU {EE4E2E03-7B4C-46E5-B9D2-89E84A18D787}.Release|Any CPU.ActiveCfg = Release|Any CPU {EE4E2E03-7B4C-46E5-B9D2-89E84A18D787}.Release|Any CPU.Build.0 = Release|Any CPU + {FC5A9799-7C44-4BFA-BA22-55DCAF1A1B9F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {FC5A9799-7C44-4BFA-BA22-55DCAF1A1B9F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {FC5A9799-7C44-4BFA-BA22-55DCAF1A1B9F}.Release|Any CPU.ActiveCfg = Release|Any CPU + {FC5A9799-7C44-4BFA-BA22-55DCAF1A1B9F}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/test/Microsoft.Sbom.Tool.Tests/IntegrationTests.cs b/test/Microsoft.Sbom.Tool.Tests/IntegrationTests.cs new file mode 100644 index 00000000..ab46fa3d --- /dev/null +++ b/test/Microsoft.Sbom.Tool.Tests/IntegrationTests.cs @@ -0,0 +1,234 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; +using System.Diagnostics; +using System.IO; +using System.Reflection; +using System.Runtime.InteropServices; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace Microsoft.Sbom.Tools.Tests; + +[TestClass] +public class IntegrationTests +{ + private const string ManifestRootFolderName = "_manifest"; + private const string ManifestFileName = "manifest.spdx.json"; + + private static readonly bool IsWindows = RuntimeInformation.IsOSPlatform(OSPlatform.Windows); + + public TestContext TestContext { get; set; } + + private static string testRunDirectory; + + [ClassInitialize] + public static void Setup(TestContext context) + { + testRunDirectory = context.TestRunDirectory; + } + + [ClassCleanup] + public static void TearDown() + { + // Clean up test directories + if (testRunDirectory is not null) + { + if (Directory.Exists(testRunDirectory)) + { + Directory.Delete(testRunDirectory, true); + } + } + } + + [TestMethod] + public void TargetAppExists() + { + Assert.IsTrue(File.Exists(GetAppName())); + } + + [TestMethod] + public void E2E_NoParameters_DisplaysHelpMessage_ReturnsNonZeroExitCode() + { + if (!IsWindows) + { + Assert.Inconclusive("This test is not (yet) supported on non-Windows platforms."); + return; + } + + var (stdout, stderr, exitCode) = LaunchAndCaptureOutput(null); + + Assert.AreEqual(stderr, string.Empty); + Assert.IsTrue(stdout.Contains("Validate -options")); + Assert.IsTrue(stdout.Contains("Generate -options")); + Assert.IsTrue(stdout.Contains("Redact -options")); + Assert.AreNotEqual(0, exitCode.Value); + } + + [TestMethod] + public void E2E_GenerateManifest_GeneratesManifest_ReturnsZeroExitCode() + { + if (!IsWindows) + { + Assert.Inconclusive("This test is not (yet) supported on non-Windows platforms."); + return; + } + + var testFolderPath = CreateTestFolder(); + GenerateManifestAndValidateSuccess(testFolderPath); + } + + [TestMethod] + public void E2E_GenerateAndValidateManifest_ValidationSucceeds_ReturnsZeroExitCode() + { + if (!IsWindows) + { + Assert.Inconclusive("This test is not (yet) supported on non-Windows platforms."); + return; + } + + var testFolderPath = CreateTestFolder(); + GenerateManifestAndValidateSuccess(testFolderPath); + + var outputFile = Path.Combine(TestContext.TestRunDirectory, TestContext.TestName, "validation.json"); + var manifestRootFolderName = Path.Combine(testFolderPath, ManifestRootFolderName); + var arguments = $"validate -m \"{manifestRootFolderName}\" -b . -o \"{outputFile}\" -mi spdx:2.2"; + + var (stdout, stderr, exitCode) = LaunchAndCaptureOutput(arguments); + + Assert.AreEqual(stderr, string.Empty); + Assert.AreEqual(0, exitCode.Value); + Assert.IsTrue(File.Exists(outputFile), $"{outputFile} should have been created during validation"); + Assert.IsTrue(File.ReadAllText(outputFile).Contains("\"Result\":\"Success\"", StringComparison.OrdinalIgnoreCase)); + } + + [TestMethod] + public void E2E_GenerateAndRedactManifest_RedactedFileIsSmaller_ReturnsZeroExitCode() + { + if (!IsWindows) + { + Assert.Inconclusive("This test is not (yet) supported on non-Windows platforms."); + return; + } + + var testFolderPath = CreateTestFolder(); + GenerateManifestAndValidateSuccess(testFolderPath); + + var outputFolder = Path.Combine(TestContext.TestRunDirectory, TestContext.TestName, "redacted"); + var originalManifestFolderPath = AppendFullManifestFolderPath(testFolderPath); + var originalManifestFilePath = Path.Combine(AppendFullManifestFolderPath(testFolderPath), ManifestFileName); + var arguments = $"redact -sp \"{originalManifestFilePath}\" -o \"{outputFolder}\" -verbosity verbose"; + + var (stdout, stderr, exitCode) = LaunchAndCaptureOutput(arguments); + + Assert.AreEqual(stderr, string.Empty); + Assert.AreEqual(0, exitCode.Value); + Assert.IsTrue(stdout.Contains("Result=Success", StringComparison.OrdinalIgnoreCase)); + var redactedManifestFilePath = Path.Combine(outputFolder, ManifestFileName); + var originalManifestSize = File.ReadAllText(originalManifestFilePath).Length; + var redactedManifestSize = File.ReadAllText(redactedManifestFilePath).Length; + Assert.IsTrue(redactedManifestSize > 0, "Redacted file must not be empty"); + Assert.IsTrue(redactedManifestSize < originalManifestSize, "Redacted file must be smaller than the original"); + } + + private void GenerateManifestAndValidateSuccess(string testFolderPath) + { + var arguments = $"generate -ps IntegrationTests -pn IntegrationTests -pv 1.2.3 -m \"{testFolderPath}\" -b . -bc \"{GetSolutionFolderPath()}\""; + + var (stdout, stderr, exitCode) = LaunchAndCaptureOutput(arguments); + + Assert.AreEqual(stderr, string.Empty); + var manifestFolderPath = AppendFullManifestFolderPath(testFolderPath); + var jsonFilePath = Path.Combine(manifestFolderPath, ManifestFileName); + var shaFilePath = Path.Combine(manifestFolderPath, "manifest.spdx.json.sha256"); + Assert.IsTrue(File.Exists(jsonFilePath)); + Assert.IsTrue(File.Exists(shaFilePath)); + Assert.AreEqual(0, exitCode.Value); + } + + private string CreateTestFolder() + { + var testFolderPath = Path.GetFullPath(Path.Combine(TestContext.TestRunDirectory, TestContext.TestName)); + Directory.CreateDirectory(testFolderPath); + return testFolderPath; + } + + private static string AppendFullManifestFolderPath(string manifestDir) + { + return Path.Combine(manifestDir, ManifestRootFolderName, "spdx_2.2"); + } + + private static string GetSolutionFolderPath() + { + return Path.GetFullPath(Path.Combine(Assembly.GetExecutingAssembly().Location, "..", "..", "..")); + } + + private static string GetAppName() + { + return IsWindows ? "Microsoft.Sbom.Tool.exe" : "Microsoft.Sbom.Tool"; + } + + private static (string stdout, string stderr, int? exitCode) LaunchAndCaptureOutput(string? arguments) + { + var stdout = string.Empty; + var stderr = string.Empty; + int? exitCode = null; + Process process = null; + + try + { + process = new Process + { + StartInfo = new ProcessStartInfo + { + FileName = GetAppName(), + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true, + Arguments = arguments ?? string.Empty, + } + }; + + process.OutputDataReceived += (sender, e) => + { + if (!string.IsNullOrEmpty(e.Data)) + { + stdout += e.Data + Environment.NewLine; + } + }; + + process.ErrorDataReceived += (sender, e) => + { + if (!string.IsNullOrEmpty(e.Data)) + { + stderr += e.Data + Environment.NewLine; + } + }; + + process.Start(); + process.BeginOutputReadLine(); + process.BeginErrorReadLine(); + process.WaitForExit(); + } + catch (Exception e) + { + Assert.Fail($"Caught the following Exception: {e}"); + } + finally + { + if (process is not null) + { + if (!process.HasExited) + { + process.Kill(); + } + + exitCode = process.ExitCode; + process.Dispose(); + } + } + + return (stdout, stderr, exitCode); + } +} diff --git a/test/Microsoft.Sbom.Tool.Tests/Microsoft.Sbom.Tool.Tests.csproj b/test/Microsoft.Sbom.Tool.Tests/Microsoft.Sbom.Tool.Tests.csproj new file mode 100644 index 00000000..0a21f494 --- /dev/null +++ b/test/Microsoft.Sbom.Tool.Tests/Microsoft.Sbom.Tool.Tests.csproj @@ -0,0 +1,18 @@ + + + + false + True + Microsoft.Sbom.Tools.Tests + $(StrongNameSigningKeyFilePath) + + + + TRACE + + + + + + +