Add simple integration tests (#606)

* Add simple integration tests * Make app name OS-aware * Remove unneeded using statements * Tweak non-Windows platforms * Missed 1 test case * Try calling chmod u+x on test app * Skip all but 1 test on non-Windows * PR feedback * Create a separate folder for each test, as intended * Use TestRunDirectory instead of TestResultsDirectory
microsoft · Jul 9, 2024 · 2f55781 · 2f55781
1 parent cdc046a
commit 2f55781
Show file tree

Hide file tree

Showing 3 changed files with 258 additions and 0 deletions.
diff --git a/Microsoft.Sbom.sln b/Microsoft.Sbom.sln
@@ -49,6 +49,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Sbom.Extensions.D
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Sbom.Extensions.DependencyInjection.Tests", "test\Microsoft.Sbom.Extensions.DependencyInjection.Tests\Microsoft.Sbom.Extensions.DependencyInjection.Tests.csproj", "{EE4E2E03-7B4C-46E5-B9D2-89E84A18D787}"
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Sbom.Tool.Tests", "test\Microsoft.Sbom.Tool.Tests\Microsoft.Sbom.Tool.Tests.csproj", "{FC5A9799-7C44-4BFA-BA22-55DCAF1A1B9F}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -107,6 +109,10 @@ Global
 		{EE4E2E03-7B4C-46E5-B9D2-89E84A18D787}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{EE4E2E03-7B4C-46E5-B9D2-89E84A18D787}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{EE4E2E03-7B4C-46E5-B9D2-89E84A18D787}.Release|Any CPU.Build.0 = Release|Any CPU
+		{FC5A9799-7C44-4BFA-BA22-55DCAF1A1B9F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{FC5A9799-7C44-4BFA-BA22-55DCAF1A1B9F}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{FC5A9799-7C44-4BFA-BA22-55DCAF1A1B9F}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{FC5A9799-7C44-4BFA-BA22-55DCAF1A1B9F}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE

diff --git a/test/Microsoft.Sbom.Tool.Tests/IntegrationTests.cs b/test/Microsoft.Sbom.Tool.Tests/IntegrationTests.cs
@@ -0,0 +1,234 @@
+// Copyright (c) Microsoft. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using System;
+using System.Diagnostics;
+using System.IO;
+using System.Reflection;
+using System.Runtime.InteropServices;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+namespace Microsoft.Sbom.Tools.Tests;
+
+[TestClass]
+public class IntegrationTests
+{
+    private const string ManifestRootFolderName = "_manifest";
+    private const string ManifestFileName = "manifest.spdx.json";
+
+    private static readonly bool IsWindows = RuntimeInformation.IsOSPlatform(OSPlatform.Windows);
+
+    public TestContext TestContext { get; set; }
+
+    private static string testRunDirectory;
+
+    [ClassInitialize]
+    public static void Setup(TestContext context)
+    {
+        testRunDirectory = context.TestRunDirectory;
+    }
+
+    [ClassCleanup]
+    public static void TearDown()
+    {
+        // Clean up test directories
+        if (testRunDirectory is not null)
+        {
+            if (Directory.Exists(testRunDirectory))
+            {
+                Directory.Delete(testRunDirectory, true);
+            }
+        }
+    }
+
+    [TestMethod]
+    public void TargetAppExists()
+    {
+        Assert.IsTrue(File.Exists(GetAppName()));
+    }
+
+    [TestMethod]
+    public void E2E_NoParameters_DisplaysHelpMessage_ReturnsNonZeroExitCode()
+    {
+        if (!IsWindows)
+        {
+            Assert.Inconclusive("This test is not (yet) supported on non-Windows platforms.");
+            return;
+        }
+
+        var (stdout, stderr, exitCode) = LaunchAndCaptureOutput(null);
+
+        Assert.AreEqual(stderr, string.Empty);
+        Assert.IsTrue(stdout.Contains("Validate -options"));
+        Assert.IsTrue(stdout.Contains("Generate -options"));
+        Assert.IsTrue(stdout.Contains("Redact -options"));
+        Assert.AreNotEqual(0, exitCode.Value);
+    }
+
+    [TestMethod]
+    public void E2E_GenerateManifest_GeneratesManifest_ReturnsZeroExitCode()
+    {
+        if (!IsWindows)
+        {
+            Assert.Inconclusive("This test is not (yet) supported on non-Windows platforms.");
+            return;
+        }
+
+        var testFolderPath = CreateTestFolder();
+        GenerateManifestAndValidateSuccess(testFolderPath);
+    }
+
+    [TestMethod]
+    public void E2E_GenerateAndValidateManifest_ValidationSucceeds_ReturnsZeroExitCode()
+    {
+        if (!IsWindows)
+        {
+            Assert.Inconclusive("This test is not (yet) supported on non-Windows platforms.");
+            return;
+        }
+
+        var testFolderPath = CreateTestFolder();
+        GenerateManifestAndValidateSuccess(testFolderPath);
+
+        var outputFile = Path.Combine(TestContext.TestRunDirectory, TestContext.TestName, "validation.json");
+        var manifestRootFolderName = Path.Combine(testFolderPath, ManifestRootFolderName);
+        var arguments = $"validate -m \"{manifestRootFolderName}\" -b . -o \"{outputFile}\" -mi spdx:2.2";
+
+        var (stdout, stderr, exitCode) = LaunchAndCaptureOutput(arguments);
+
+        Assert.AreEqual(stderr, string.Empty);
+        Assert.AreEqual(0, exitCode.Value);
+        Assert.IsTrue(File.Exists(outputFile), $"{outputFile} should have been created during validation");
+        Assert.IsTrue(File.ReadAllText(outputFile).Contains("\"Result\":\"Success\"", StringComparison.OrdinalIgnoreCase));
+    }
+
+    [TestMethod]
+    public void E2E_GenerateAndRedactManifest_RedactedFileIsSmaller_ReturnsZeroExitCode()
+    {
+        if (!IsWindows)
+        {
+            Assert.Inconclusive("This test is not (yet) supported on non-Windows platforms.");
+            return;
+        }
+
+        var testFolderPath = CreateTestFolder();
+        GenerateManifestAndValidateSuccess(testFolderPath);
+
+        var outputFolder = Path.Combine(TestContext.TestRunDirectory, TestContext.TestName, "redacted");
+        var originalManifestFolderPath = AppendFullManifestFolderPath(testFolderPath);
+        var originalManifestFilePath = Path.Combine(AppendFullManifestFolderPath(testFolderPath), ManifestFileName);
+        var arguments = $"redact -sp \"{originalManifestFilePath}\" -o \"{outputFolder}\" -verbosity verbose";
+
+        var (stdout, stderr, exitCode) = LaunchAndCaptureOutput(arguments);
+
+        Assert.AreEqual(stderr, string.Empty);
+        Assert.AreEqual(0, exitCode.Value);
+        Assert.IsTrue(stdout.Contains("Result=Success", StringComparison.OrdinalIgnoreCase));
+        var redactedManifestFilePath = Path.Combine(outputFolder, ManifestFileName);
+        var originalManifestSize = File.ReadAllText(originalManifestFilePath).Length;
+        var redactedManifestSize = File.ReadAllText(redactedManifestFilePath).Length;
+        Assert.IsTrue(redactedManifestSize > 0, "Redacted file must not be empty");
+        Assert.IsTrue(redactedManifestSize < originalManifestSize, "Redacted file must be smaller than the original");
+    }
+
+    private void GenerateManifestAndValidateSuccess(string testFolderPath)
+    {
+        var arguments = $"generate -ps IntegrationTests -pn IntegrationTests -pv 1.2.3 -m \"{testFolderPath}\"  -b . -bc \"{GetSolutionFolderPath()}\"";
+
+        var (stdout, stderr, exitCode) = LaunchAndCaptureOutput(arguments);
+
+        Assert.AreEqual(stderr, string.Empty);
+        var manifestFolderPath = AppendFullManifestFolderPath(testFolderPath);
+        var jsonFilePath = Path.Combine(manifestFolderPath, ManifestFileName);
+        var shaFilePath = Path.Combine(manifestFolderPath, "manifest.spdx.json.sha256");
+        Assert.IsTrue(File.Exists(jsonFilePath));
+        Assert.IsTrue(File.Exists(shaFilePath));
+        Assert.AreEqual(0, exitCode.Value);
+    }
+
+    private string CreateTestFolder()
+    {
+        var testFolderPath = Path.GetFullPath(Path.Combine(TestContext.TestRunDirectory, TestContext.TestName));
+        Directory.CreateDirectory(testFolderPath);
+        return testFolderPath;
+    }
+
+    private static string AppendFullManifestFolderPath(string manifestDir)
+    {
+        return Path.Combine(manifestDir, ManifestRootFolderName, "spdx_2.2");
+    }
+
+    private static string GetSolutionFolderPath()
+    {
+        return Path.GetFullPath(Path.Combine(Assembly.GetExecutingAssembly().Location, "..", "..", ".."));
+    }
+
+    private static string GetAppName()
+    {
+        return IsWindows ? "Microsoft.Sbom.Tool.exe" : "Microsoft.Sbom.Tool";
+    }
+
+    private static (string stdout, string stderr, int? exitCode) LaunchAndCaptureOutput(string? arguments)
+    {
+        var stdout = string.Empty;
+        var stderr = string.Empty;
+        int? exitCode = null;
+        Process process = null;
+
+        try
+        {
+            process = new Process
+            {
+                StartInfo = new ProcessStartInfo
+                {
+                    FileName = GetAppName(),
+                    RedirectStandardOutput = true,
+                    RedirectStandardError = true,
+                    UseShellExecute = false,
+                    CreateNoWindow = true,
+                    Arguments = arguments ?? string.Empty,
+                }
+            };
+
+            process.OutputDataReceived += (sender, e) =>
+            {
+                if (!string.IsNullOrEmpty(e.Data))
+                {
+                    stdout += e.Data + Environment.NewLine;
+                }
+            };
+
+            process.ErrorDataReceived += (sender, e) =>
+            {
+                if (!string.IsNullOrEmpty(e.Data))
+                {
+                    stderr += e.Data + Environment.NewLine;
+                }
+            };
+
+            process.Start();
+            process.BeginOutputReadLine();
+            process.BeginErrorReadLine();
+            process.WaitForExit();
+        }
+        catch (Exception e)
+        {
+            Assert.Fail($"Caught the following Exception: {e}");
+        }
+        finally
+        {
+            if (process is not null)
+            {
+                if (!process.HasExited)
+                {
+                    process.Kill();
+                }
+
+                exitCode = process.ExitCode;
+                process.Dispose();
+            }
+        }
+
+        return (stdout, stderr, exitCode);
+    }
+}
diff --git a/test/Microsoft.Sbom.Tool.Tests/Microsoft.Sbom.Tool.Tests.csproj b/test/Microsoft.Sbom.Tool.Tests/Microsoft.Sbom.Tool.Tests.csproj
@@ -0,0 +1,18 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <IsPackable>false</IsPackable>
+    <SignAssembly>True</SignAssembly>
+    <RootNamespace>Microsoft.Sbom.Tools.Tests</RootNamespace>
+    <AssemblyOriginatorKeyFile>$(StrongNameSigningKeyFilePath)</AssemblyOriginatorKeyFile>
+  </PropertyGroup>
+
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|AnyCPU'">
+    <DefineConstants>TRACE</DefineConstants>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\src\Microsoft.Sbom.Tool\Microsoft.Sbom.Tool.csproj" />
+  </ItemGroup>
+
+</Project>