Merge branch 'main' into security-severity

ReleaseHistory.md

@@ -1,4 +1,8 @@
 # SARIF Package Release History (SDK, Driver, Converters, and Multitool)
+## **v4.3.1** UNRELEASED
+* BUG: Improve `HdfConverter` ensure uri data is populated and to provide location and region data property from `SourceLocation`. [#2704](https://github.com/microsoft/sarif-sdk/pull/2704)
+* BUG: Correct `run.language` regex in JSON schema. [#2708]https://github.com/microsoft/sarif-sdk/pull/2708
+
 ## **v4.3.0** [Sdk](https://www.nuget.org/packages/Sarif.Sdk/v4.3.0) | [Driver](https://www.nuget.org/packages/Sarif.Driver/v4.3.0) | [Converters](https://www.nuget.org/packages/Sarif.Converters/v4.3.0) | [Multitool](https://www.nuget.org/packages/Sarif.Multitool/v4.3.0) | [Multitool Library](https://www.nuget.org/packages/Sarif.Multitool.Library/v4.3.0)
 * BUG: Resolve `NullReferenceException` retrieving `MultithreadedZipArchiveArtifactProvider.SizeInBytes` after content have been faulted in.
 

src/Sarif.Converters/HdfConverter.cs

@@ -173,14 +173,14 @@ private static (ReportingDescriptor, IList<Result>) SarifRuleAndResultFromHdfCon
                             {
                                 ArtifactLocation = new ArtifactLocation
                                 {
-                                   Uri =  new Uri(".", UriKind.Relative),
+                                   Uri =  new Uri(execJsonControl.SourceLocation.Ref ?? "file:///", UriKind.RelativeOrAbsolute),
                                    UriBaseId = "ROOTPATH",
                                 },
                                 Region = new Region
                                 {
-                                    StartLine = 1,
+                                    StartLine = execJsonControl.SourceLocation.Line ?? 1,
                                     StartColumn = 1,
-                                    EndLine = 1,
+                                    EndLine = execJsonControl.SourceLocation.Line ?? 1,
                                     EndColumn = 1,
                                 }
                             }

src/Sarif.Converters/HdfModel/SourceLocation.cs

@@ -10,13 +10,13 @@ public partial class SourceLocation
         /// <summary>
         /// The line at which this statement is located in the file
         /// </summary>
-        [JsonProperty("line", Required = Required.Default)]
-        public double Line { get; set; }
+        [JsonProperty("line", Required = Required.DisallowNull, NullValueHandling = NullValueHandling.Ignore)]
+        public int? Line { get; set; }
 
         /// <summary>
         /// Path to the file that this statement originates from
         /// </summary>
-        [JsonProperty("ref", Required = Required.Default)]
+        [JsonProperty("ref", Required = Required.DisallowNull, NullValueHandling = NullValueHandling.Ignore)]
         public string Ref { get; set; }
     }
 }

src/Sarif.Driver/Sdk/AnalyzeOptionsBase.cs

@@ -36,6 +36,11 @@ public abstract class AnalyzeOptionsBase : CommonOptionsBase
             HelpText = "Path to policy file that will be used to configure analysis. This defaults to 'default.configuration.xml' beside the main tool; passing value of 'default' or removing that file will configure the tool to use its built-in settings.")]
         public string ConfigurationFilePath { get; set; }
 
+        [Option(
+            "output-config",
+            HelpText = "Path to a policy file to which all analysis settings from the current run will be saved.")]
+        public string OutputConfigurationFilePath { get; set; }
+
         [Option(
             'q',
             "quiet",

src/Sarif.Driver/Sdk/CommonOptionsBase.cs

@@ -110,7 +110,7 @@ internal static IEnumerable<FilePersistenceOptions> NormalizeFilePersistenceOpti
         [Option(
             "automation-guid",
             HelpText = "A guid that will be persisted to the 'Run.AutomationDetails.Guid' property. See section '3.17.4' of the SARIF specification for more information.")]
-        public Guid? AutomationGuid { get; set; }
+        public Guid AutomationGuid { get; set; }
 
         public Formatting Formatting => this.PrettyPrint || (!this.PrettyPrint && !this.Minify)
             ? Formatting.Indented

src/Sarif.Driver/Sdk/MultithreadedAnalyzeCommandBase.cs

@@ -11,6 +11,7 @@
 using System.Linq;
 using System.Net;
 using System.Net.Http;
+using System.Reflection;
 using System.Runtime.InteropServices;
 using System.Threading.Channels;
 using System.Threading.Tasks;
@@ -293,22 +294,22 @@ public virtual TContext InitializeGlobalContextFromOptions(TOptions options, ref
             context.Logger ??= InitializeLogger(context);
 
             // Finally, handle the remaining options.
-
+            context.PostUri = options.PostUri ?? context.PostUri;
             context.AutomationId = options.AutomationId ?? context.AutomationId;
             context.Threads = options.Threads > 0 ? options.Threads : context.Threads;
-            context.AutomationGuid = options.AutomationGuid ?? context.AutomationGuid;
+            context.AutomationGuid = options.AutomationGuid != default ? options.AutomationGuid : context.AutomationGuid;
             context.OutputFilePath = options.OutputFilePath ?? context.OutputFilePath;
-            context.EventsFilePath = Environment.GetEnvironmentVariable("SPMI_ETW") ?? options.EventsFilePath ?? context.EventsFilePath;
-            context.PostUri = options.PostUri != null ? options.PostUri : context.PostUri;
+            context.BaselineFilePath = options.BaselineFilePath ?? context.BaselineFilePath;
             context.Recurse = options.Recurse != null ? options.Recurse.Value : context.Recurse;
             context.Traces = options.Trace.Any() ? InitializeStringSet(options.Trace) : context.Traces;
-            context.BaselineFilePath = options.BaselineFilePath != null ? options.BaselineFilePath : context.BaselineFilePath;
+            context.GlobalFilePathDenyRegex = options.GlobalFilePathDenyRegex ?? context.GlobalFilePathDenyRegex;
+            context.OutputConfigurationFilePath = options.OutputConfigurationFilePath ?? context.OutputConfigurationFilePath;
             context.DataToInsert = options.DataToInsert?.Any() == true ? options.DataToInsert.ToFlags() : context.DataToInsert;
             context.DataToRemove = options.DataToRemove?.Any() == true ? options.DataToRemove.ToFlags() : context.DataToRemove;
+            context.EventsFilePath = Environment.GetEnvironmentVariable("SPMI_ETW") ?? options.EventsFilePath ?? context.EventsFilePath;
             context.OutputFileOptions = options.OutputFileOptions?.Any() == true ? options.OutputFileOptions.ToFlags() : context.OutputFileOptions;
             context.PluginFilePaths = options.PluginFilePaths?.Any() == true ? options.PluginFilePaths?.ToImmutableHashSet() : context.PluginFilePaths;
             context.InsertProperties = options.InsertProperties?.Any() == true ? InitializeStringSet(options.InsertProperties) : context.InsertProperties;
-            context.GlobalFilePathDenyRegex = options.GlobalFilePathDenyRegex != null ? options.GlobalFilePathDenyRegex : context.GlobalFilePathDenyRegex;
             context.MaxFileSizeInKilobytes = options.MaxFileSizeInKilobytes != null ? options.MaxFileSizeInKilobytes.Value : context.MaxFileSizeInKilobytes;
             context.TargetFileSpecifiers = options.TargetFileSpecifiers?.Any() == true ? InitializeStringSet(options.TargetFileSpecifiers) : context.TargetFileSpecifiers;
             context.InvocationPropertiesToLog = options.InvocationPropertiesToLog?.Any() == true ? InitializeStringSet(options.InvocationPropertiesToLog) : context.InvocationPropertiesToLog;
@@ -824,12 +825,25 @@ internal string GetConfigurationFileName(string configurationFilePath, IFileSyst
                     : null;
             }
 
+            if (!File.Exists(configurationFilePath))
+            {
+                string fileName = Path.GetFileNameWithoutExtension(configurationFilePath);
+                string spamDirectory = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location);
+                fileName = Path.Combine(spamDirectory, $"{fileName}.xml");
+
+                if (fileSystem.FileExists(fileName))
+                {
+                    return fileName;
+                }
+            }
+
             return configurationFilePath;
         }
 
         protected virtual TContext InitializeConfiguration(string configurationFileName, TContext context)
         {
             context.Policy ??= new PropertiesDictionary();
+
             configurationFileName = GetConfigurationFileName(configurationFileName, context.FileSystem);
             context.ConfigurationFilePath = configurationFileName;
 

src/Sarif.Multitool.Library/ValidateCommand.cs

@@ -1,7 +1,6 @@
 // Copyright (c) Microsoft. All rights reserved.
 // Licensed under the MIT license. See LICENSE file in the project root for full license information.
 
-using System;
 using System.Collections.Generic;
 using System.IO;
 using System.Reflection;

src/Sarif/AnalyzeContextBase.cs

@@ -29,6 +29,7 @@ public virtual IEnumerable<IOption> GetOptions()
                 AutomationIdProperty,
                 BaselineFilePathProperty,
                 ChannelSizeProperty,
+                OutputConfigurationFilePathProperty,
                 DataToInsertProperty,
                 DataToRemoveProperty,
                 EventsFilePathProperty,
@@ -112,7 +113,7 @@ public virtual int ChannelSize
             set => this.Policy.SetProperty(ChannelSizeProperty, value);
         }
 
-        public virtual Guid? AutomationGuid
+        public virtual Guid AutomationGuid
         {
             get => this.Policy.GetProperty(AutomationGuidProperty);
             set => this.Policy.SetProperty(AutomationGuidProperty, value);
@@ -142,10 +143,12 @@ public string OutputFilePath
             set => this.Policy.SetProperty(OutputFilePathProperty, value);
         }
 
-        public string ConfigurationFilePath
+        public string ConfigurationFilePath { get; set; }
+
+        public string OutputConfigurationFilePath
         {
-            get => this.Policy.GetProperty(ConfigurationFilePathProperty);
-            set => this.Policy.SetProperty(ConfigurationFilePathProperty, value);
+            get => this.Policy.GetProperty(OutputConfigurationFilePathProperty);
+            set => this.Policy.SetProperty(OutputConfigurationFilePathProperty, value);
         }
 
         public string EventsFilePath
@@ -272,9 +275,9 @@ public virtual void Dispose()
                 "CoreSettings", nameof(ChannelSize), defaultValue: () => 50000,
                 "The capacity of the channels for analyzing scan targets and logging results.");
 
-        public static PerLanguageOption<Guid?> AutomationGuidProperty { get; } =
-            new PerLanguageOption<Guid?>(
-                "CoreSettings", nameof(AutomationGuid), defaultValue: () => null,
+        public static PerLanguageOption<Guid> AutomationGuidProperty { get; } =
+            new PerLanguageOption<Guid>(
+                "CoreSettings", nameof(AutomationGuid), defaultValue: () => default,
                 "A guid that will be persisted to the 'Run.AutomationDetails.Guid' property. " +
                 "See section '3.17.4' of the SARIF specification for more information.");
 
@@ -304,10 +307,10 @@ public virtual void Dispose()
                                 "CoreSettings", nameof(PostUri), defaultValue: () => string.Empty,
                                 "A SARIF-accepting endpoint to publish the output log to.");
 
-        public static PerLanguageOption<string> ConfigurationFilePathProperty { get; } =
+        public static PerLanguageOption<string> OutputConfigurationFilePathProperty { get; } =
                             new PerLanguageOption<string>(
-                                "CoreSettings", nameof(ConfigurationFilePath), defaultValue: () => string.Empty,
-                                "The path to write all SARIF log file results to.");
+                                "CoreSettings", nameof(OutputConfigurationFilePath), defaultValue: () => string.Empty,
+                                "The path to write all resolved configuration (by current command-line) to.");
 
         public static PerLanguageOption<OptionallyEmittedData> DataToInsertProperty { get; } =
                     new PerLanguageOption<OptionallyEmittedData>(

src/Sarif/Core/Run.cs

@@ -224,8 +224,8 @@ public bool ShouldSerializeAutomationDetails()
         {
             return this.AutomationDetails?.Description != null ||
                 !string.IsNullOrWhiteSpace(this.AutomationDetails?.Id) ||
-                this.AutomationDetails?.Guid != null ||
-                this.AutomationDetails?.CorrelationGuid != null;
+                (this.AutomationDetails?.Guid != null && this.AutomationDetails.Guid.Value != Guid.Empty) ||
+                (this.AutomationDetails?.CorrelationGuid != null && this.AutomationDetails.CorrelationGuid != Guid.Empty);
         }
 
         public bool ShouldSerializeInvocations()

src/Sarif/IAnalysisContext.cs

@@ -27,7 +27,7 @@ public interface IAnalysisContext : IDisposable
 
         public string AutomationId { get; set; }
 
-        public Guid? AutomationGuid { get; set; }
+        public Guid AutomationGuid { get; set; }
 
         FilePersistenceOptions OutputFileOptions { get; set; }
 

src/Sarif/PropertiesDictionary.cs

@@ -201,6 +201,11 @@ public void LoadFromXml(Stream stream)
         {
             this.Clear();
 
+            if (stream.CanSeek)
+            {
+                stream.Seek(0, SeekOrigin.Begin);
+            }
+
             var settings = new XmlReaderSettings
             {
                 DtdProcessing = DtdProcessing.Ignore,

src/Sarif/PropertiesDictionaryExtensionMethods.cs

@@ -45,7 +45,7 @@ public static void SavePropertiesToXmlStream(
             propertyBagType = propertyBag.GetType();
             propertyBagTypeName = propertyBagType.Name;
 
-            if (propertyBagTypeName != "PropertyBag")
+            if (propertyBagTypeName != "PropertiesDictionary")
             {
                 propertyBagTypeName = NormalizeTypeName(propertyBag.GetType().FullName);
             }
@@ -56,7 +56,7 @@ public static void SavePropertiesToXmlStream(
                 writer.WriteAttributeString(KEY_ID, name);
             }
 
-            if (propertyBagTypeName != "PropertyBag")
+            if (propertyBagTypeName != "PropertiesDictionary")
             {
                 writer.WriteAttributeString(TYPE_ID, propertyBagTypeName);
             }

src/Sarif/Schemata/sarif-2.1.0-rtm.6.json

@@ -2354,7 +2354,7 @@
           "description": "The language of the messages emitted into the log file during this run (expressed as an ISO 639-1 two-letter lowercase culture code) and an optional region (expressed as an ISO 3166-1 two-letter uppercase subculture code associated with a country or region). The casing is recommended but not required (in order for this data to conform to RFC5646).",
           "type": "string",
           "default": "en-US",
-          "pattern": "^[a-zA-Z]{2}|^[a-zA-Z]{2}-[a-zA-Z]{2}]?$"
+          "pattern": "^[a-zA-Z]{2}(-[a-zA-Z]{2})?$"
         },
 
         "versionControlProvenance": {