From 4073642df5983a721b665053a1b1bb8a042c90ec Mon Sep 17 00:00:00 2001 From: "Michael C. Fanning" Date: Tue, 1 Aug 2023 13:38:23 -0700 Subject: [PATCH] Add realized configuration peristence mechanism. Namespace removal. (#2706) * Add realized configuration peristence mechanism. Namespace removal. * Restore tests to working order. * Test fix. --- src/Sarif.Driver/Sdk/AnalyzeOptionsBase.cs | 5 ++++ src/Sarif.Driver/Sdk/CommonOptionsBase.cs | 2 +- .../Sdk/MultithreadedAnalyzeCommandBase.cs | 26 ++++++++++++++----- .../ValidateCommand.cs | 1 - src/Sarif/AnalyzeContextBase.cs | 23 +++++++++------- src/Sarif/Core/Run.cs | 4 +-- src/Sarif/IAnalysisContext.cs | 2 +- src/Sarif/PropertiesDictionary.cs | 5 ++++ .../PropertiesDictionaryExtensionMethods.cs | 4 +-- .../Sdk/AnalyzeCommandBaseTests.cs | 22 ++++++++-------- src/Test.Utilities.Sarif/DirectoryHelpers.cs | 1 - 11 files changed, 60 insertions(+), 35 deletions(-) diff --git a/src/Sarif.Driver/Sdk/AnalyzeOptionsBase.cs b/src/Sarif.Driver/Sdk/AnalyzeOptionsBase.cs index 2e59e8d33..496181bfd 100644 --- a/src/Sarif.Driver/Sdk/AnalyzeOptionsBase.cs +++ b/src/Sarif.Driver/Sdk/AnalyzeOptionsBase.cs @@ -36,6 +36,11 @@ public abstract class AnalyzeOptionsBase : CommonOptionsBase HelpText = "Path to policy file that will be used to configure analysis. This defaults to 'default.configuration.xml' beside the main tool; passing value of 'default' or removing that file will configure the tool to use its built-in settings.")] public string ConfigurationFilePath { get; set; } + [Option( + "output-config", + HelpText = "Path to a policy file to which all analysis settings from the current run will be saved.")] + public string OutputConfigurationFilePath { get; set; } + [Option( 'q', "quiet", diff --git a/src/Sarif.Driver/Sdk/CommonOptionsBase.cs b/src/Sarif.Driver/Sdk/CommonOptionsBase.cs index 20e90d0c1..f3c095010 100644 --- a/src/Sarif.Driver/Sdk/CommonOptionsBase.cs +++ b/src/Sarif.Driver/Sdk/CommonOptionsBase.cs @@ -110,7 +110,7 @@ internal static IEnumerable NormalizeFilePersistenceOpti [Option( "automation-guid", HelpText = "A guid that will be persisted to the 'Run.AutomationDetails.Guid' property. See section '3.17.4' of the SARIF specification for more information.")] - public Guid? AutomationGuid { get; set; } + public Guid AutomationGuid { get; set; } public Formatting Formatting => this.PrettyPrint || (!this.PrettyPrint && !this.Minify) ? Formatting.Indented diff --git a/src/Sarif.Driver/Sdk/MultithreadedAnalyzeCommandBase.cs b/src/Sarif.Driver/Sdk/MultithreadedAnalyzeCommandBase.cs index 220f86dc1..bce732073 100644 --- a/src/Sarif.Driver/Sdk/MultithreadedAnalyzeCommandBase.cs +++ b/src/Sarif.Driver/Sdk/MultithreadedAnalyzeCommandBase.cs @@ -11,6 +11,7 @@ using System.Linq; using System.Net; using System.Net.Http; +using System.Reflection; using System.Runtime.InteropServices; using System.Threading.Channels; using System.Threading.Tasks; @@ -293,22 +294,22 @@ public virtual TContext InitializeGlobalContextFromOptions(TOptions options, ref context.Logger ??= InitializeLogger(context); // Finally, handle the remaining options. - + context.PostUri = options.PostUri ?? context.PostUri; context.AutomationId = options.AutomationId ?? context.AutomationId; context.Threads = options.Threads > 0 ? options.Threads : context.Threads; - context.AutomationGuid = options.AutomationGuid ?? context.AutomationGuid; + context.AutomationGuid = options.AutomationGuid != default ? options.AutomationGuid : context.AutomationGuid; context.OutputFilePath = options.OutputFilePath ?? context.OutputFilePath; - context.EventsFilePath = Environment.GetEnvironmentVariable("SPMI_ETW") ?? options.EventsFilePath ?? context.EventsFilePath; - context.PostUri = options.PostUri != null ? options.PostUri : context.PostUri; + context.BaselineFilePath = options.BaselineFilePath ?? context.BaselineFilePath; context.Recurse = options.Recurse != null ? options.Recurse.Value : context.Recurse; context.Traces = options.Trace.Any() ? InitializeStringSet(options.Trace) : context.Traces; - context.BaselineFilePath = options.BaselineFilePath != null ? options.BaselineFilePath : context.BaselineFilePath; + context.GlobalFilePathDenyRegex = options.GlobalFilePathDenyRegex ?? context.GlobalFilePathDenyRegex; + context.OutputConfigurationFilePath = options.OutputConfigurationFilePath ?? context.OutputConfigurationFilePath; context.DataToInsert = options.DataToInsert?.Any() == true ? options.DataToInsert.ToFlags() : context.DataToInsert; context.DataToRemove = options.DataToRemove?.Any() == true ? options.DataToRemove.ToFlags() : context.DataToRemove; + context.EventsFilePath = Environment.GetEnvironmentVariable("SPMI_ETW") ?? options.EventsFilePath ?? context.EventsFilePath; context.OutputFileOptions = options.OutputFileOptions?.Any() == true ? options.OutputFileOptions.ToFlags() : context.OutputFileOptions; context.PluginFilePaths = options.PluginFilePaths?.Any() == true ? options.PluginFilePaths?.ToImmutableHashSet() : context.PluginFilePaths; context.InsertProperties = options.InsertProperties?.Any() == true ? InitializeStringSet(options.InsertProperties) : context.InsertProperties; - context.GlobalFilePathDenyRegex = options.GlobalFilePathDenyRegex != null ? options.GlobalFilePathDenyRegex : context.GlobalFilePathDenyRegex; context.MaxFileSizeInKilobytes = options.MaxFileSizeInKilobytes != null ? options.MaxFileSizeInKilobytes.Value : context.MaxFileSizeInKilobytes; context.TargetFileSpecifiers = options.TargetFileSpecifiers?.Any() == true ? InitializeStringSet(options.TargetFileSpecifiers) : context.TargetFileSpecifiers; context.InvocationPropertiesToLog = options.InvocationPropertiesToLog?.Any() == true ? InitializeStringSet(options.InvocationPropertiesToLog) : context.InvocationPropertiesToLog; @@ -824,12 +825,25 @@ internal string GetConfigurationFileName(string configurationFilePath, IFileSyst : null; } + if (!File.Exists(configurationFilePath)) + { + string fileName = Path.GetFileNameWithoutExtension(configurationFilePath); + string spamDirectory = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location); + fileName = Path.Combine(spamDirectory, $"{fileName}.xml"); + + if (fileSystem.FileExists(fileName)) + { + return fileName; + } + } + return configurationFilePath; } protected virtual TContext InitializeConfiguration(string configurationFileName, TContext context) { context.Policy ??= new PropertiesDictionary(); + configurationFileName = GetConfigurationFileName(configurationFileName, context.FileSystem); context.ConfigurationFilePath = configurationFileName; diff --git a/src/Sarif.Multitool.Library/ValidateCommand.cs b/src/Sarif.Multitool.Library/ValidateCommand.cs index 1bede8e0d..4ecd0c1d2 100644 --- a/src/Sarif.Multitool.Library/ValidateCommand.cs +++ b/src/Sarif.Multitool.Library/ValidateCommand.cs @@ -1,7 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. -using System; using System.Collections.Generic; using System.IO; using System.Reflection; diff --git a/src/Sarif/AnalyzeContextBase.cs b/src/Sarif/AnalyzeContextBase.cs index a2dc87463..fa6cdc8ce 100644 --- a/src/Sarif/AnalyzeContextBase.cs +++ b/src/Sarif/AnalyzeContextBase.cs @@ -29,6 +29,7 @@ public virtual IEnumerable GetOptions() AutomationIdProperty, BaselineFilePathProperty, ChannelSizeProperty, + OutputConfigurationFilePathProperty, DataToInsertProperty, DataToRemoveProperty, EventsFilePathProperty, @@ -112,7 +113,7 @@ public virtual int ChannelSize set => this.Policy.SetProperty(ChannelSizeProperty, value); } - public virtual Guid? AutomationGuid + public virtual Guid AutomationGuid { get => this.Policy.GetProperty(AutomationGuidProperty); set => this.Policy.SetProperty(AutomationGuidProperty, value); @@ -142,10 +143,12 @@ public string OutputFilePath set => this.Policy.SetProperty(OutputFilePathProperty, value); } - public string ConfigurationFilePath + public string ConfigurationFilePath { get; set; } + + public string OutputConfigurationFilePath { - get => this.Policy.GetProperty(ConfigurationFilePathProperty); - set => this.Policy.SetProperty(ConfigurationFilePathProperty, value); + get => this.Policy.GetProperty(OutputConfigurationFilePathProperty); + set => this.Policy.SetProperty(OutputConfigurationFilePathProperty, value); } public string EventsFilePath @@ -272,9 +275,9 @@ public virtual void Dispose() "CoreSettings", nameof(ChannelSize), defaultValue: () => 50000, "The capacity of the channels for analyzing scan targets and logging results."); - public static PerLanguageOption AutomationGuidProperty { get; } = - new PerLanguageOption( - "CoreSettings", nameof(AutomationGuid), defaultValue: () => null, + public static PerLanguageOption AutomationGuidProperty { get; } = + new PerLanguageOption( + "CoreSettings", nameof(AutomationGuid), defaultValue: () => default, "A guid that will be persisted to the 'Run.AutomationDetails.Guid' property. " + "See section '3.17.4' of the SARIF specification for more information."); @@ -304,10 +307,10 @@ public virtual void Dispose() "CoreSettings", nameof(PostUri), defaultValue: () => string.Empty, "A SARIF-accepting endpoint to publish the output log to."); - public static PerLanguageOption ConfigurationFilePathProperty { get; } = + public static PerLanguageOption OutputConfigurationFilePathProperty { get; } = new PerLanguageOption( - "CoreSettings", nameof(ConfigurationFilePath), defaultValue: () => string.Empty, - "The path to write all SARIF log file results to."); + "CoreSettings", nameof(OutputConfigurationFilePath), defaultValue: () => string.Empty, + "The path to write all resolved configuration (by current command-line) to."); public static PerLanguageOption DataToInsertProperty { get; } = new PerLanguageOption( diff --git a/src/Sarif/Core/Run.cs b/src/Sarif/Core/Run.cs index 443291a56..03706e907 100644 --- a/src/Sarif/Core/Run.cs +++ b/src/Sarif/Core/Run.cs @@ -224,8 +224,8 @@ public bool ShouldSerializeAutomationDetails() { return this.AutomationDetails?.Description != null || !string.IsNullOrWhiteSpace(this.AutomationDetails?.Id) || - this.AutomationDetails?.Guid != null || - this.AutomationDetails?.CorrelationGuid != null; + (this.AutomationDetails?.Guid != null && this.AutomationDetails.Guid.Value != Guid.Empty) || + (this.AutomationDetails?.CorrelationGuid != null && this.AutomationDetails.CorrelationGuid != Guid.Empty); } public bool ShouldSerializeInvocations() diff --git a/src/Sarif/IAnalysisContext.cs b/src/Sarif/IAnalysisContext.cs index f569a1ee8..7bbc318b4 100644 --- a/src/Sarif/IAnalysisContext.cs +++ b/src/Sarif/IAnalysisContext.cs @@ -27,7 +27,7 @@ public interface IAnalysisContext : IDisposable public string AutomationId { get; set; } - public Guid? AutomationGuid { get; set; } + public Guid AutomationGuid { get; set; } FilePersistenceOptions OutputFileOptions { get; set; } diff --git a/src/Sarif/PropertiesDictionary.cs b/src/Sarif/PropertiesDictionary.cs index 62623402d..09314b9da 100644 --- a/src/Sarif/PropertiesDictionary.cs +++ b/src/Sarif/PropertiesDictionary.cs @@ -201,6 +201,11 @@ public void LoadFromXml(Stream stream) { this.Clear(); + if (stream.CanSeek) + { + stream.Seek(0, SeekOrigin.Begin); + } + var settings = new XmlReaderSettings { DtdProcessing = DtdProcessing.Ignore, diff --git a/src/Sarif/PropertiesDictionaryExtensionMethods.cs b/src/Sarif/PropertiesDictionaryExtensionMethods.cs index 9c560d0e2..123ea19d3 100644 --- a/src/Sarif/PropertiesDictionaryExtensionMethods.cs +++ b/src/Sarif/PropertiesDictionaryExtensionMethods.cs @@ -45,7 +45,7 @@ public static void SavePropertiesToXmlStream( propertyBagType = propertyBag.GetType(); propertyBagTypeName = propertyBagType.Name; - if (propertyBagTypeName != "PropertyBag") + if (propertyBagTypeName != "PropertiesDictionary") { propertyBagTypeName = NormalizeTypeName(propertyBag.GetType().FullName); } @@ -56,7 +56,7 @@ public static void SavePropertiesToXmlStream( writer.WriteAttributeString(KEY_ID, name); } - if (propertyBagTypeName != "PropertyBag") + if (propertyBagTypeName != "PropertiesDictionary") { writer.WriteAttributeString(TYPE_ID, propertyBagTypeName); } diff --git a/src/Test.UnitTests.Sarif.Driver/Sdk/AnalyzeCommandBaseTests.cs b/src/Test.UnitTests.Sarif.Driver/Sdk/AnalyzeCommandBaseTests.cs index dcf5dc5fa..219ac9cca 100644 --- a/src/Test.UnitTests.Sarif.Driver/Sdk/AnalyzeCommandBaseTests.cs +++ b/src/Test.UnitTests.Sarif.Driver/Sdk/AnalyzeCommandBaseTests.cs @@ -1190,12 +1190,12 @@ public void AnalyzeCommandBase_EndToEndAnalysisWithExplicitlyDisabledRules() } [Theory] - [InlineData(null, false, "")] - [InlineData("", false, "")] + [InlineData(null, false, null)] + [InlineData("", false, null)] [InlineData(null, true, "default.configuration.xml")] [InlineData("", true, "default.configuration.xml")] - [InlineData("default", false, "")] - [InlineData("default", true, "")] + [InlineData("default", false, null)] + [InlineData("default", true, null)] [InlineData("test-newconfig.xml", false, "test-newconfig.xml")] [InlineData("test-newconfig.xml", true, "test-newconfig.xml")] public void AnalyzeCommandBase_LoadConfigurationFile(string configValue, bool defaultFileExists, string expectedFileName) @@ -1220,7 +1220,7 @@ public void AnalyzeCommandBase_LoadConfigurationFile(string configValue, bool de if (string.IsNullOrEmpty(expectedFileName)) { - context.ConfigurationFilePath.Should().Be(string.Empty); + context.ConfigurationFilePath.Should().Be(null); } else { @@ -1520,7 +1520,7 @@ public void AnalyzeCommandBase_AutomationDetailsTests() new TestAnalyzeOptions { AutomationId = string.Empty, - AutomationGuid = null + AutomationGuid = default }, new TestAnalyzeOptions { @@ -1530,7 +1530,7 @@ public void AnalyzeCommandBase_AutomationDetailsTests() new TestAnalyzeOptions { AutomationId = null, - AutomationGuid = null + AutomationGuid = default }, new TestAnalyzeOptions { @@ -1540,7 +1540,7 @@ public void AnalyzeCommandBase_AutomationDetailsTests() new TestAnalyzeOptions { AutomationId = string.Empty, - AutomationGuid = null + AutomationGuid = default }, new TestAnalyzeOptions { @@ -1823,7 +1823,7 @@ private static void RunResultsCachingTestCase(ResultsCachingTestCase testCase, { runWithCaching.Artifacts.Should().NotBeEmpty(); - if (string.IsNullOrWhiteSpace(options.AutomationId) && options.AutomationGuid == null) + if (string.IsNullOrWhiteSpace(options.AutomationId) && options.AutomationGuid == default) { runWithCaching.AutomationDetails.Should().Be(null); } @@ -1833,7 +1833,7 @@ private static void RunResultsCachingTestCase(ResultsCachingTestCase testCase, runWithCaching.AutomationDetails.Id.Should().Be(options.AutomationId); } - if (options.AutomationGuid != null) + if (options.AutomationGuid != default) { runWithCaching.AutomationDetails.Guid.Should().Be(options.AutomationGuid); } @@ -1852,7 +1852,7 @@ private static void RunResultsCachingTestCase(ResultsCachingTestCase testCase, private static void EnhanceOptions(TestAnalyzeOptions current, TestAnalyzeOptions enhancement) { current.AutomationId ??= enhancement?.AutomationId; - current.AutomationGuid ??= enhancement?.AutomationGuid; + current.AutomationGuid = enhancement == null ? default : enhancement.AutomationGuid; } private static IFileSystem CreateDefaultFileSystemForResultsCaching(IList files, bool generateSameInput = false) diff --git a/src/Test.Utilities.Sarif/DirectoryHelpers.cs b/src/Test.Utilities.Sarif/DirectoryHelpers.cs index 878862f0d..771323073 100644 --- a/src/Test.Utilities.Sarif/DirectoryHelpers.cs +++ b/src/Test.Utilities.Sarif/DirectoryHelpers.cs @@ -1,7 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. -using System; using System.IO; using System.Reflection;