From 273d48eb61f8a2a92eb6c51a1393beee8d5039f9 Mon Sep 17 00:00:00 2001 From: Justin Perez Date: Thu, 23 Feb 2023 12:28:08 -0800 Subject: [PATCH] feat: Switch to Serilog (#441) --- .editorconfig | 3 + Directory.Packages.props | 88 +++--- .../ComponentStreamEnumerable.cs | 15 +- .../ComponentStreamEnumerableFactory.cs | 6 +- .../DependencyGraph/ComponentRecorder.cs | 25 +- .../DockerService.cs | 6 +- .../FastDirectoryWalkerFactory.cs | 14 +- .../LazyComponentStream.cs | 11 +- .../Logger.cs | 165 ----------- ...Microsoft.ComponentDetection.Common.csproj | 1 + .../PathUtilityService.cs | 13 +- .../SafeFileEnumerable.cs | 6 +- .../SafeFileEnumerableFactory.cs | 6 +- .../Telemetry/CommandLineTelemetryService.cs | 9 +- .../FileComponentDetector.cs | 4 +- .../ILogger.cs | 58 ---- ...rosoft.ComponentDetection.Contracts.csproj | 11 +- .../ScanRequest.cs | 1 + ...rosoft.ComponentDetection.Detectors.csproj | 3 +- .../cocoapods/PodComponentDetector.cs | 10 +- .../dockerfile/DockerfileComponentDetector.cs | 12 +- .../go/GoComponentDetector.cs | 30 +- .../gradle/GradleComponentDetector.cs | 6 +- .../ivy/IvyDetector.cs | 35 ++- .../linux/LinuxContainerDetector.cs | 22 +- .../linux/LinuxScanner.cs | 10 +- .../maven/MavenCommandService.cs | 12 +- .../maven/MvnCliComponentDetector.cs | 8 +- .../npm/NpmComponentDetector.cs | 16 +- .../npm/NpmComponentDetectorWithRoots.cs | 16 +- .../npm/NpmComponentUtilities.cs | 5 +- .../nuget/NuGetComponentDetector.cs | 13 +- .../nuget/NuGetPackagesConfigDetector.cs | 5 +- ...ectModelProjectCentricComponentDetector.cs | 19 +- .../pip/IPyPiClient.cs | 31 +- .../pip/PipComponentDetector.cs | 8 +- .../pip/PythonResolver.cs | 21 +- .../pnpm/PnpmComponentDetector.cs | 10 +- .../poetry/PoetryComponentDetector.cs | 6 +- .../ruby/RubyComponentDetector.cs | 12 +- .../rust/RustCrateDetector.cs | 8 +- .../spdx/Spdx22ComponentDetector.cs | 18 +- .../vcpkg/VcpkgComponentDetector.cs | 12 +- .../yarn/IYarnLockFileFactory.cs | 1 + .../yarn/IYarnLockParser.cs | 2 + .../yarn/Parsers/YarnLockParser.cs | 10 +- .../yarn/YarnLockComponentDetector.cs | 21 +- .../yarn/YarnLockFileFactory.cs | 2 + .../Extensions/ServiceCollectionExtensions.cs | 24 +- ...oft.ComponentDetection.Orchestrator.csproj | 8 +- .../Orchestrator.cs | 50 +++- .../Services/BcdeDevCommandService.cs | 11 +- .../Services/BcdeScanCommandService.cs | 15 +- .../Services/BcdeScanExecutionService.cs | 12 +- .../Services/DetectorListingCommandService.cs | 14 +- .../Services/DetectorProcessingService.cs | 40 +-- .../Services/DetectorRestrictionService.cs | 7 +- .../DefaultGraphTranslationService.cs | 13 +- .../Services/ServiceBase.cs | 7 - .../Microsoft.ComponentDetection.csproj | 4 + src/Microsoft.ComponentDetection/Program.cs | 8 + .../ComponentStreamEnumerableTests.cs | 11 +- .../DockerServiceTests.cs | 5 +- .../FileEnumerationTests.cs | 5 +- .../LoggerTests.cs | 272 ------------------ ...oft.ComponentDetection.Common.Tests.csproj | 5 +- .../SafeFileEnumerableTests.cs | 2 + ....ComponentDetection.Contracts.Tests.csproj | 9 +- .../LinuxContainerDetectorTests.cs | 30 +- .../LinuxScannerTests.cs | 6 +- .../MavenCommandServiceTests.cs | 4 +- ....ComponentDetection.Detectors.Tests.csproj | 1 + .../NpmUtilitiesTests.cs | 4 +- .../NuGetComponentDetectorTests.cs | 22 +- .../PipComponentDetectorTests.cs | 18 +- .../PipResolverTests.cs | 6 +- .../PnpmDetectorTests.cs | 3 + .../PyPiClientTests.cs | 15 +- .../YarnLockDetectorTests.cs | 6 +- .../YarnParserTests.cs | 6 +- ...mponentDetection.Orchestrator.Tests.csproj | 7 +- .../Services/BcdeDevCommandServiceTests.cs | 5 +- .../Services/BcdeScanExecutionServiceTests.cs | 13 +- .../DetectorListingCommandServiceTests.cs | 28 +- .../DetectorProcessingServiceTests.cs | 5 +- .../DetectorRestrictionServiceTests.cs | 6 +- .../DetectorTestUtilityBuilder.cs | 5 +- ...t.ComponentDetection.TestsUtilities.csproj | 1 + .../ComponentDetectionIntegrationTests.cs | 2 +- ...pendencyDetective.VerificationTests.csproj | 14 +- 90 files changed, 651 insertions(+), 904 deletions(-) delete mode 100644 src/Microsoft.ComponentDetection.Common/Logger.cs delete mode 100644 src/Microsoft.ComponentDetection.Contracts/ILogger.cs delete mode 100644 src/Microsoft.ComponentDetection.Orchestrator/Services/ServiceBase.cs delete mode 100644 test/Microsoft.ComponentDetection.Common.Tests/LoggerTests.cs diff --git a/.editorconfig b/.editorconfig index 709d39b81..56ed3ca0d 100644 --- a/.editorconfig +++ b/.editorconfig @@ -669,6 +669,9 @@ dotnet_diagnostic.CA1852.severity = suggestion # CA1854: Prefer the IDictionary.TryGetValue(TKey, out TValue) method dotnet_diagnostic.CA1854.severity = suggestion +# CA1848: Use the LoggerMessage delegates +dotnet_diagnostic.CA1848.severity = suggestion + # JSON002: Probable JSON string detected dotnet_diagnostic.JSON002.severity = suggestion diff --git a/Directory.Packages.props b/Directory.Packages.props index a7bf05e29..dafbe6936 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -1,44 +1,48 @@ - - - - - Compile - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + Compile + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/Microsoft.ComponentDetection.Common/ComponentStreamEnumerable.cs b/src/Microsoft.ComponentDetection.Common/ComponentStreamEnumerable.cs index d4c0ba728..7afd066fb 100644 --- a/src/Microsoft.ComponentDetection.Common/ComponentStreamEnumerable.cs +++ b/src/Microsoft.ComponentDetection.Common/ComponentStreamEnumerable.cs @@ -1,29 +1,31 @@ namespace Microsoft.ComponentDetection.Common; + using System; using System.Collections; using System.Collections.Generic; using System.IO; using Microsoft.ComponentDetection.Contracts; +using Microsoft.Extensions.Logging; public class ComponentStreamEnumerable : IEnumerable { + private readonly ILogger logger; + public ComponentStreamEnumerable(IEnumerable fileEnumerable, ILogger logger) { + this.logger = logger; this.ToEnumerate = fileEnumerable; - this.Logger = logger; } private IEnumerable ToEnumerate { get; } - private ILogger Logger { get; } - public IEnumerator GetEnumerator() { foreach (var filePairing in this.ToEnumerate) { if (!filePairing.File.Exists) { - this.Logger.LogWarning($"File {filePairing.File.FullName} does not exist on disk."); + this.logger.LogWarning("File {FilePairingName} does not exist on disk.", filePairing.File.FullName); yield break; } @@ -51,13 +53,12 @@ private Stream SafeOpenFile(FileInfo file) } catch (UnauthorizedAccessException) { - this.Logger.LogWarning($"Unauthorized access exception caught when trying to open {file.FullName}"); + this.logger.LogWarning("Unauthorized access exception caught when trying to open {FileName}", file.FullName); return null; } catch (Exception e) { - this.Logger.LogWarning($"Unhandled exception caught when trying to open {file.FullName}"); - this.Logger.LogException(e, isError: false); + this.logger.LogWarning(e, "Unhandled exception caught when trying to open {FileName}", file.FullName); return null; } } diff --git a/src/Microsoft.ComponentDetection.Common/ComponentStreamEnumerableFactory.cs b/src/Microsoft.ComponentDetection.Common/ComponentStreamEnumerableFactory.cs index 0639dbf5b..810b1c2c9 100644 --- a/src/Microsoft.ComponentDetection.Common/ComponentStreamEnumerableFactory.cs +++ b/src/Microsoft.ComponentDetection.Common/ComponentStreamEnumerableFactory.cs @@ -1,15 +1,17 @@ namespace Microsoft.ComponentDetection.Common; + using System; using System.Collections.Generic; using System.IO; using Microsoft.ComponentDetection.Contracts; +using Microsoft.Extensions.Logging; public class ComponentStreamEnumerableFactory : IComponentStreamEnumerableFactory { private readonly IPathUtilityService pathUtilityService; - private readonly ILogger logger; + private readonly ILogger logger; - public ComponentStreamEnumerableFactory(IPathUtilityService pathUtilityService, ILogger logger) + public ComponentStreamEnumerableFactory(IPathUtilityService pathUtilityService, ILogger logger) { this.pathUtilityService = pathUtilityService; this.logger = logger; diff --git a/src/Microsoft.ComponentDetection.Common/DependencyGraph/ComponentRecorder.cs b/src/Microsoft.ComponentDetection.Common/DependencyGraph/ComponentRecorder.cs index 8b35d226e..71b1270df 100644 --- a/src/Microsoft.ComponentDetection.Common/DependencyGraph/ComponentRecorder.cs +++ b/src/Microsoft.ComponentDetection.Common/DependencyGraph/ComponentRecorder.cs @@ -12,17 +12,19 @@ namespace Microsoft.ComponentDetection.Common.DependencyGraph; +using Microsoft.Extensions.Logging; + public class ComponentRecorder : IComponentRecorder { - private readonly ILogger log; - private readonly ConcurrentBag singleFileRecorders = new ConcurrentBag(); private readonly bool enableManualTrackingOfExplicitReferences; - public ComponentRecorder(ILogger log = null, bool enableManualTrackingOfExplicitReferences = true) + private readonly ILogger logger; + + public ComponentRecorder(ILogger logger = null, bool enableManualTrackingOfExplicitReferences = true) { - this.log = log; + this.logger = logger; this.enableManualTrackingOfExplicitReferences = enableManualTrackingOfExplicitReferences; } @@ -86,7 +88,7 @@ public ISingleFileComponentRecorder CreateSingleFileComponentRecorder(string loc var matching = this.singleFileRecorders.FirstOrDefault(x => x.ManifestFileLocation == location); if (matching == null) { - matching = new SingleFileComponentRecorder(location, this, this.enableManualTrackingOfExplicitReferences, this.log); + matching = new SingleFileComponentRecorder(location, this, this.enableManualTrackingOfExplicitReferences, this.logger); this.singleFileRecorders.Add(matching); } @@ -106,8 +108,6 @@ internal DependencyGraph GetDependencyGraphForLocation(string location) public sealed class SingleFileComponentRecorder : ISingleFileComponentRecorder { - private readonly ILogger log; - private readonly ConcurrentDictionary detectedComponentsInternal = new ConcurrentDictionary(); /// @@ -116,14 +116,15 @@ public sealed class SingleFileComponentRecorder : ISingleFileComponentRecorder private readonly ConcurrentDictionary skippedComponentsInternal = new ConcurrentDictionary(); private readonly ComponentRecorder recorder; + private readonly ILogger logger; private readonly object registerUsageLock = new object(); - public SingleFileComponentRecorder(string location, ComponentRecorder recorder, bool enableManualTrackingOfExplicitReferences, ILogger log) + public SingleFileComponentRecorder(string location, ComponentRecorder recorder, bool enableManualTrackingOfExplicitReferences, ILogger logger) { this.ManifestFileLocation = location; this.recorder = recorder; - this.log = log; + this.logger = logger; this.DependencyGraph = new DependencyGraph(enableManualTrackingOfExplicitReferences); } @@ -174,17 +175,17 @@ public void RegisterUsage( #if DEBUG if (detectedComponent.FilePaths?.Any() ?? false) { - this.log?.LogWarning("Detector should not populate DetectedComponent.FilePaths!"); + this.logger.LogWarning("Detector should not populate DetectedComponent.FilePaths!"); } if (detectedComponent.DependencyRoots?.Any() ?? false) { - this.log?.LogWarning("Detector should not populate DetectedComponent.DependencyRoots!"); + this.logger.LogWarning("Detector should not populate DetectedComponent.DependencyRoots!"); } if (detectedComponent.DevelopmentDependency.HasValue) { - this.log?.LogWarning("Detector should not populate DetectedComponent.DevelopmentDependency!"); + this.logger.LogWarning("Detector should not populate DetectedComponent.DevelopmentDependency!"); } #endif diff --git a/src/Microsoft.ComponentDetection.Common/DockerService.cs b/src/Microsoft.ComponentDetection.Common/DockerService.cs index aab6333b9..906384648 100644 --- a/src/Microsoft.ComponentDetection.Common/DockerService.cs +++ b/src/Microsoft.ComponentDetection.Common/DockerService.cs @@ -1,4 +1,5 @@ namespace Microsoft.ComponentDetection.Common; + using System; using System.Collections.Generic; using System.IO; @@ -10,6 +11,7 @@ namespace Microsoft.ComponentDetection.Common; using Microsoft.ComponentDetection.Common.Telemetry.Records; using Microsoft.ComponentDetection.Contracts; using Microsoft.ComponentDetection.Contracts.BcdeModels; +using Microsoft.Extensions.Logging; using Newtonsoft.Json; public class DockerService : IDockerService @@ -23,7 +25,7 @@ public class DockerService : IDockerService private readonly ILogger logger; - public DockerService(ILogger logger) => this.logger = logger; + public DockerService(ILogger logger) => this.logger = logger; public async Task CanPingDockerAsync(CancellationToken cancellationToken = default) { @@ -34,7 +36,7 @@ public async Task CanPingDockerAsync(CancellationToken cancellationToken = } catch (Exception e) { - this.logger.LogException(e, false); + this.logger.LogError(e, "Failed to ping docker"); return false; } } diff --git a/src/Microsoft.ComponentDetection.Common/FastDirectoryWalkerFactory.cs b/src/Microsoft.ComponentDetection.Common/FastDirectoryWalkerFactory.cs index d1eecf146..718c3bd91 100644 --- a/src/Microsoft.ComponentDetection.Common/FastDirectoryWalkerFactory.cs +++ b/src/Microsoft.ComponentDetection.Common/FastDirectoryWalkerFactory.cs @@ -1,4 +1,5 @@ namespace Microsoft.ComponentDetection.Common; + using System; using System.Collections.Concurrent; using System.Collections.Generic; @@ -12,14 +13,15 @@ using System.Threading.Tasks.Dataflow; using Microsoft.ComponentDetection.Contracts; using Microsoft.ComponentDetection.Contracts.Internal; +using Microsoft.Extensions.Logging; public class FastDirectoryWalkerFactory : IObservableDirectoryWalkerFactory { private readonly ConcurrentDictionary>> pendingScans = new ConcurrentDictionary>>(); private readonly IPathUtilityService pathUtilityService; - private readonly ILogger logger; + private readonly ILogger logger; - public FastDirectoryWalkerFactory(IPathUtilityService pathUtilityService, ILogger logger) + public FastDirectoryWalkerFactory(IPathUtilityService pathUtilityService, ILogger logger) { this.pathUtilityService = pathUtilityService; this.logger = logger; @@ -31,7 +33,7 @@ public IObservable GetDirectoryScanner(DirectoryInfo root, Concu { if (!root.Exists) { - this.logger?.LogError($"Root directory doesn't exist: {root.FullName}"); + this.logger.LogError("Root directory doesn't exist: {RootFullName}", root.FullName); s.OnCompleted(); return Task.CompletedTask; } @@ -49,7 +51,7 @@ public IObservable GetDirectoryScanner(DirectoryInfo root, Concu var sw = Stopwatch.StartNew(); - this.logger?.LogInfo($"Starting enumeration of {root.FullName}"); + this.logger.LogInformation("Starting enumeration of {RootFullName}", root.FullName); var fileCount = 0; var directoryCount = 0; @@ -187,7 +189,7 @@ public IObservable GetDirectoryScanner(DirectoryInfo root, Concu () => { sw.Stop(); - this.logger?.LogInfo($"Enumerated {fileCount} files and {directoryCount} directories in {sw.Elapsed}"); + this.logger.LogInformation("Enumerated {FileCount} files and {DirectoryCount} directories in {Elapsed}", fileCount, directoryCount, sw.Elapsed); s.OnCompleted(); }); }); @@ -211,7 +213,7 @@ public IObservable Subscribe(DirectoryInfo root, IEnumerable { diff --git a/src/Microsoft.ComponentDetection.Common/LazyComponentStream.cs b/src/Microsoft.ComponentDetection.Common/LazyComponentStream.cs index b0ac4d6a9..4ab8bd39c 100644 --- a/src/Microsoft.ComponentDetection.Common/LazyComponentStream.cs +++ b/src/Microsoft.ComponentDetection.Common/LazyComponentStream.cs @@ -1,13 +1,15 @@ namespace Microsoft.ComponentDetection.Common; + using System; using System.IO; using Microsoft.ComponentDetection.Contracts; +using Microsoft.Extensions.Logging; public class LazyComponentStream : IComponentStream { private readonly FileInfo fileInfo; - private readonly Lazy fileBuffer; private readonly ILogger logger; + private readonly Lazy fileBuffer; public LazyComponentStream(FileInfo fileInfo, string pattern, ILogger logger) { @@ -35,14 +37,13 @@ private byte[] SafeOpenFile() return buffer; } - catch (UnauthorizedAccessException) + catch (UnauthorizedAccessException e) { - this.logger?.LogWarning($"Unauthorized access exception caught when trying to open {this.fileInfo.FullName}"); + this.logger.LogWarning(e, "Unauthorized access exception caught when trying to open {FileName}", this.fileInfo.FullName); } catch (Exception e) { - this.logger?.LogWarning($"Unhandled exception caught when trying to open {this.fileInfo.FullName}"); - this.logger?.LogException(e, isError: false); + this.logger.LogWarning(e, "Unhandled exception caught when trying to open {FileName}", this.fileInfo.FullName); } return Array.Empty(); diff --git a/src/Microsoft.ComponentDetection.Common/Logger.cs b/src/Microsoft.ComponentDetection.Common/Logger.cs deleted file mode 100644 index 51a33264f..000000000 --- a/src/Microsoft.ComponentDetection.Common/Logger.cs +++ /dev/null @@ -1,165 +0,0 @@ -namespace Microsoft.ComponentDetection.Common; -using System; -using System.Runtime.CompilerServices; -using Microsoft.ComponentDetection.Common.Telemetry.Records; -using Microsoft.ComponentDetection.Contracts; - -using static System.Environment; - -public class Logger : ILogger -{ - public const string LogRelativePath = "GovCompDisc_Log_{timestamp}.log"; - - private readonly IConsoleWritingService consoleWriter; - private readonly IFileWritingService fileWritingService; - - public Logger(IConsoleWritingService consoleWriter, IFileWritingService fileWritingService) - { - this.consoleWriter = consoleWriter; - this.fileWritingService = fileWritingService; - } - - private VerbosityMode Verbosity { get; set; } - - private bool Initialized { get; set; } - - private bool WriteToFile { get; set; } - - private bool WriteLinePrefix { get; set; } - - public void Init(VerbosityMode verbosity, bool writeLinePrefix = true) - { - this.WriteToFile = true; - this.Verbosity = verbosity; - this.WriteLinePrefix = writeLinePrefix; - - // If initialization has already completed, don't attempt to create - // the log file again as this throws an exception - if (this.Initialized) - { - return; - } - - try - { - this.fileWritingService.WriteFile(LogRelativePath, string.Empty); - this.LogInfo($"Log file: {this.fileWritingService.ResolveFilePath(LogRelativePath)}"); - this.Initialized = true; - } - catch (Exception) - { - this.WriteToFile = false; - this.LogError("There was an issue writing to the log file, for the remainder of execution verbose output will be written to the console."); - this.Verbosity = VerbosityMode.Verbose; - } - } - - public void LogCreateLoggingGroup() - { - this.PrintToConsole(NewLine, VerbosityMode.Normal); - this.AppendToFile(NewLine); - } - - public void LogWarning(string message) - { - this.LogInternal("WARN", message); - } - - public void LogInfo(string message) - { - this.LogInternal("INFO", message); - } - - public void LogVerbose(string message) - { - this.LogInternal("VERBOSE", message, VerbosityMode.Verbose); - } - - public void LogError(string message) - { - this.LogInternal("ERROR", message, VerbosityMode.Quiet); - } - - public void LogFailedReadingFile(string filePath, Exception e) - { - this.PrintToConsole(NewLine, VerbosityMode.Verbose); - this.LogFailedProcessingFile(filePath); - this.LogException(e, isError: false); - using var record = new FailedReadingFileRecord - { - FilePath = filePath, - ExceptionMessage = e.Message, - StackTrace = e.StackTrace, - }; - } - - public void LogException( - Exception e, - bool isError, - bool printException = false, - [CallerMemberName] string callerMemberName = "", - [CallerLineNumber] int callerLineNumber = 0) - { - var tag = isError ? "[ERROR]" : "[INFO]"; - - var fullExceptionText = $"{tag} Exception encountered." + NewLine + - $"CallerMember: [{callerMemberName} : {callerLineNumber}]" + NewLine + - e.ToString() + NewLine; - - var shortExceptionText = $"{tag} {callerMemberName} logged {e.GetType().Name}: {e.Message}{NewLine}"; - - var consoleText = printException ? fullExceptionText : shortExceptionText; - - if (isError) - { - this.PrintToConsole(consoleText, VerbosityMode.Quiet); - } - else - { - this.PrintToConsole(consoleText, VerbosityMode.Verbose); - } - - this.AppendToFile(fullExceptionText); - } - - // TODO: All these vso specific logs should go away - public void LogBuildWarning(string message) - { - this.PrintToConsole($"##vso[task.LogIssue type=warning;]{message}{NewLine}", VerbosityMode.Quiet); - } - - public void LogBuildError(string message) - { - this.PrintToConsole($"##vso[task.LogIssue type=error;]{message}{NewLine}", VerbosityMode.Quiet); - } - - private void LogFailedProcessingFile(string filePath) - { - this.LogVerbose($"Could not read component details from file {filePath} {NewLine}"); - } - - private void AppendToFile(string text) - { - if (this.WriteToFile) - { - this.fileWritingService.AppendToFile(LogRelativePath, text); - } - } - - private void PrintToConsole(string text, VerbosityMode minVerbosity) - { - if (this.Verbosity >= minVerbosity) - { - this.consoleWriter.Write(text); - } - } - - private void LogInternal(string prefix, string message, VerbosityMode verbosity = VerbosityMode.Normal) - { - var formattedPrefix = (!this.WriteLinePrefix || string.IsNullOrWhiteSpace(prefix)) ? string.Empty : $"[{prefix}] "; - var text = $"{formattedPrefix}{message} {NewLine}"; - - this.PrintToConsole(text, verbosity); - this.AppendToFile(text); - } -} diff --git a/src/Microsoft.ComponentDetection.Common/Microsoft.ComponentDetection.Common.csproj b/src/Microsoft.ComponentDetection.Common/Microsoft.ComponentDetection.Common.csproj index 17a59f724..c32f070cc 100644 --- a/src/Microsoft.ComponentDetection.Common/Microsoft.ComponentDetection.Common.csproj +++ b/src/Microsoft.ComponentDetection.Common/Microsoft.ComponentDetection.Common.csproj @@ -3,6 +3,7 @@ + diff --git a/src/Microsoft.ComponentDetection.Common/PathUtilityService.cs b/src/Microsoft.ComponentDetection.Common/PathUtilityService.cs index 2b3af51cb..46cc066ab 100644 --- a/src/Microsoft.ComponentDetection.Common/PathUtilityService.cs +++ b/src/Microsoft.ComponentDetection.Common/PathUtilityService.cs @@ -1,4 +1,5 @@ namespace Microsoft.ComponentDetection.Common; + using System; using System.Collections.Concurrent; using System.Diagnostics; @@ -7,6 +8,7 @@ using System.Runtime.InteropServices; using System.Text; using Microsoft.ComponentDetection.Contracts; +using Microsoft.Extensions.Logging; using Microsoft.Win32.SafeHandles; // We may want to consider breaking this class into Win/Mac/Linux variants if it gets bigger @@ -26,16 +28,11 @@ public class PathUtilityService : IPathUtilityService private readonly ConcurrentDictionary resolvedPaths = new ConcurrentDictionary(); - private readonly ILogger logger; - private readonly object isRunningOnWindowsContainerLock = new object(); + private readonly ILogger logger; private bool? isRunningOnWindowsContainer; - public PathUtilityService() - { - } - - public PathUtilityService(ILogger logger) => this.logger = logger; + public PathUtilityService(ILogger logger) => this.logger = logger; public bool IsRunningOnWindowsContainer { @@ -222,7 +219,7 @@ public string ResolvePhysicalPathLibC(string path) } catch (Exception ex) { - this.logger.LogException(ex, isError: false, printException: true); + this.logger.LogError(ex, "Failed to resolve path {Path}", path); return path; } finally diff --git a/src/Microsoft.ComponentDetection.Common/SafeFileEnumerable.cs b/src/Microsoft.ComponentDetection.Common/SafeFileEnumerable.cs index 71b920965..27458d776 100644 --- a/src/Microsoft.ComponentDetection.Common/SafeFileEnumerable.cs +++ b/src/Microsoft.ComponentDetection.Common/SafeFileEnumerable.cs @@ -1,17 +1,18 @@ namespace Microsoft.ComponentDetection.Common; + using System; using System.Collections; using System.Collections.Generic; using System.IO; using System.IO.Enumeration; using Microsoft.ComponentDetection.Contracts; +using Microsoft.Extensions.Logging; public class SafeFileEnumerable : IEnumerable { private readonly IEnumerable searchPatterns; private readonly ExcludeDirectoryPredicate directoryExclusionPredicate; private readonly DirectoryInfo directory; - private readonly ILogger logger; private readonly IPathUtilityService pathUtilityService; private readonly bool recursivelyScanDirectories; private readonly Func fileMatchingPredicate; @@ -19,6 +20,7 @@ public class SafeFileEnumerable : IEnumerable private readonly EnumerationOptions enumerationOptions; private readonly HashSet enumeratedDirectories; + private readonly ILogger logger; public SafeFileEnumerable(DirectoryInfo directory, IEnumerable searchPatterns, ILogger logger, IPathUtilityService pathUtilityService, ExcludeDirectoryPredicate directoryExclusionPredicate, bool recursivelyScanDirectories = true, HashSet previouslyEnumeratedDirectories = null) { @@ -117,7 +119,7 @@ public IEnumerator GetEnumerator() if (seenPreviously) { - this.logger.LogVerbose($"Encountered real path {targetPath} before. Short-Circuiting directory traversal"); + this.logger.LogDebug("Encountered real path {TargetPath} before. Short-Circuiting directory traversal", targetPath); return false; } diff --git a/src/Microsoft.ComponentDetection.Common/SafeFileEnumerableFactory.cs b/src/Microsoft.ComponentDetection.Common/SafeFileEnumerableFactory.cs index 83ac2bc11..6e67c065f 100644 --- a/src/Microsoft.ComponentDetection.Common/SafeFileEnumerableFactory.cs +++ b/src/Microsoft.ComponentDetection.Common/SafeFileEnumerableFactory.cs @@ -1,18 +1,20 @@ namespace Microsoft.ComponentDetection.Common; + using System.Collections.Generic; using System.IO; using Microsoft.ComponentDetection.Contracts; +using Microsoft.Extensions.Logging; public class SafeFileEnumerableFactory : ISafeFileEnumerableFactory { private readonly IPathUtilityService pathUtilityService; - private readonly ILogger logger; + private readonly ILogger logger; public SafeFileEnumerableFactory() { } - public SafeFileEnumerableFactory(IPathUtilityService pathUtilityService, ILogger logger) + public SafeFileEnumerableFactory(IPathUtilityService pathUtilityService, ILogger logger) { this.pathUtilityService = pathUtilityService; this.logger = logger; diff --git a/src/Microsoft.ComponentDetection.Common/Telemetry/CommandLineTelemetryService.cs b/src/Microsoft.ComponentDetection.Common/Telemetry/CommandLineTelemetryService.cs index 6f02c79c5..450a04f73 100644 --- a/src/Microsoft.ComponentDetection.Common/Telemetry/CommandLineTelemetryService.cs +++ b/src/Microsoft.ComponentDetection.Common/Telemetry/CommandLineTelemetryService.cs @@ -1,8 +1,9 @@ namespace Microsoft.ComponentDetection.Common.Telemetry; + using System; using System.Collections.Concurrent; using Microsoft.ComponentDetection.Common.Telemetry.Records; -using Microsoft.ComponentDetection.Contracts; +using Microsoft.Extensions.Logging; using Newtonsoft.Json; using Newtonsoft.Json.Linq; @@ -12,12 +13,12 @@ internal class CommandLineTelemetryService : ITelemetryService public const string TelemetryRelativePath = "ScanTelemetry_{timestamp}.json"; - private readonly IFileWritingService fileWritingService; private readonly ILogger logger; + private readonly IFileWritingService fileWritingService; private TelemetryMode telemetryMode = TelemetryMode.Production; - public CommandLineTelemetryService(ILogger logger, IFileWritingService fileWritingService) + public CommandLineTelemetryService(ILogger logger, IFileWritingService fileWritingService) { this.logger = logger; this.fileWritingService = fileWritingService; @@ -40,7 +41,7 @@ public void PostRecord(IDetectionTelemetryRecord record) if (this.telemetryMode == TelemetryMode.Debug) { - this.logger.LogInfo(jsonRecord.ToString()); + this.logger.LogInformation("Telemetry record: {Record}", jsonRecord.ToString()); } } } diff --git a/src/Microsoft.ComponentDetection.Contracts/FileComponentDetector.cs b/src/Microsoft.ComponentDetection.Contracts/FileComponentDetector.cs index 6976d1913..1bc1ae4d6 100644 --- a/src/Microsoft.ComponentDetection.Contracts/FileComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Contracts/FileComponentDetector.cs @@ -1,4 +1,5 @@ namespace Microsoft.ComponentDetection.Contracts; + using System; using System.Collections.Generic; using System.IO; @@ -7,6 +8,7 @@ using System.Threading.Tasks.Dataflow; using Microsoft.ComponentDetection.Contracts.Internal; using Microsoft.ComponentDetection.Contracts.TypedComponent; +using Microsoft.Extensions.Logging; /// Specialized base class for file based component detection. public abstract class FileComponentDetector : IComponentDetector @@ -71,7 +73,7 @@ private Task ScanDirectoryAsync(ScanRequest reques var filteredObservable = this.Scanner.GetFilteredComponentStreamObservable(request.SourceDirectory, this.SearchPatterns, request.ComponentRecorder); - this.Logger?.LogVerbose($"Registered {this.GetType().FullName}"); + this.Logger.LogDebug("Registered {Detector}", this.GetType().FullName); return this.ProcessAsync(filteredObservable, request.DetectorArgs); } diff --git a/src/Microsoft.ComponentDetection.Contracts/ILogger.cs b/src/Microsoft.ComponentDetection.Contracts/ILogger.cs deleted file mode 100644 index 4e2d04a0e..000000000 --- a/src/Microsoft.ComponentDetection.Contracts/ILogger.cs +++ /dev/null @@ -1,58 +0,0 @@ -namespace Microsoft.ComponentDetection.Contracts; -using System; -using System.Runtime.CompilerServices; - -/// Simple abstraction around console/output file logging for component detection. -public interface ILogger -{ - void Init(VerbosityMode verbosity, bool writeLinePrefix = true); - - /// Creates a logical separation (e.g. newline) between different log messages. - void LogCreateLoggingGroup(); - - /// Logs a warning message, outputting if configured verbosity is higher than Quiet. - /// The message to output. - void LogWarning(string message); - - /// Logs an informational message, outputting if configured verbosity is higher than Quiet. - /// The message to output. - void LogInfo(string message); - - /// Logs a verbose message, outputting if configured verbosity is at least Verbose. - /// The message to output. - void LogVerbose(string message); - - /// Logs an error message, outputting for all verbosity levels. - /// The message to output. - void LogError(string message); - - /// Logs a specially formatted message if a file read failed, outputting if configured verbosity is at least Verbose. - /// The file path responsible for the file reading failure. - /// The exception encountered when reading a file. - void LogFailedReadingFile(string filePath, Exception e); - - /// Logs a specially formatted message if an exception has occurred. - /// The exception to log the occurance of. - /// Whether or not the exception represents a true error case (e.g. unexpected) vs. expected. - /// Indicate if the exception is going to be fully printed. - /// Implicity populated arg, provides the member name of the calling function to the log message. - /// Implicitly populated arg, provides calling line number. - void LogException( - Exception e, - bool isError, - bool printException = false, - [CallerMemberName] string callerMemberName = "", - [CallerLineNumber] int callerLineNumber = 0); - - /// - /// Log a warning to the build console, adding it to the build summary and turning the build yellow. - /// - /// The message to display alongside the warning. - void LogBuildWarning(string message); - - /// - /// Log an error to the build console, adding it to the build summary and turning the build red. - /// - /// The message to display alongside the warning. - void LogBuildError(string message); -} diff --git a/src/Microsoft.ComponentDetection.Contracts/Microsoft.ComponentDetection.Contracts.csproj b/src/Microsoft.ComponentDetection.Contracts/Microsoft.ComponentDetection.Contracts.csproj index 4b0828e9a..67cf4bf2c 100644 --- a/src/Microsoft.ComponentDetection.Contracts/Microsoft.ComponentDetection.Contracts.csproj +++ b/src/Microsoft.ComponentDetection.Contracts/Microsoft.ComponentDetection.Contracts.csproj @@ -5,11 +5,12 @@ - - - - - + + + + + + diff --git a/src/Microsoft.ComponentDetection.Contracts/ScanRequest.cs b/src/Microsoft.ComponentDetection.Contracts/ScanRequest.cs index c635c6c85..8d4748290 100644 --- a/src/Microsoft.ComponentDetection.Contracts/ScanRequest.cs +++ b/src/Microsoft.ComponentDetection.Contracts/ScanRequest.cs @@ -1,6 +1,7 @@ namespace Microsoft.ComponentDetection.Contracts; using System.Collections.Generic; using System.IO; +using Microsoft.Extensions.Logging; /// Request object for a component scan. public class ScanRequest diff --git a/src/Microsoft.ComponentDetection.Detectors/Microsoft.ComponentDetection.Detectors.csproj b/src/Microsoft.ComponentDetection.Detectors/Microsoft.ComponentDetection.Detectors.csproj index 8772c507b..f51c4be23 100644 --- a/src/Microsoft.ComponentDetection.Detectors/Microsoft.ComponentDetection.Detectors.csproj +++ b/src/Microsoft.ComponentDetection.Detectors/Microsoft.ComponentDetection.Detectors.csproj @@ -2,6 +2,7 @@ + @@ -12,7 +13,7 @@ - + diff --git a/src/Microsoft.ComponentDetection.Detectors/cocoapods/PodComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/cocoapods/PodComponentDetector.cs index bc79d666a..86f3ec2e6 100644 --- a/src/Microsoft.ComponentDetection.Detectors/cocoapods/PodComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/cocoapods/PodComponentDetector.cs @@ -1,4 +1,5 @@ namespace Microsoft.ComponentDetection.Detectors.CocoaPods; + using System; using System.Collections.Generic; using System.IO; @@ -8,6 +9,7 @@ namespace Microsoft.ComponentDetection.Detectors.CocoaPods; using Microsoft.ComponentDetection.Contracts; using Microsoft.ComponentDetection.Contracts.Internal; using Microsoft.ComponentDetection.Contracts.TypedComponent; +using Microsoft.Extensions.Logging; using YamlDotNet.Core; using YamlDotNet.Core.Events; using YamlDotNet.Serialization; @@ -17,7 +19,7 @@ public class PodComponentDetector : FileComponentDetector public PodComponentDetector( IComponentStreamEnumerableFactory componentStreamEnumerableFactory, IObservableDirectoryWalkerFactory walkerFactory, - ILogger logger) + ILogger logger) { this.ComponentStreamEnumerableFactory = componentStreamEnumerableFactory; this.Scanner = walkerFactory; @@ -39,7 +41,7 @@ protected override async Task OnFileFoundAsync(ProcessRequest processRequest, ID var singleFileComponentRecorder = processRequest.SingleFileComponentRecorder; var file = processRequest.ComponentStream; - this.Logger.LogVerbose($"Found {file.Pattern}: {file.Location}"); + this.Logger.LogDebug("Found {Pattern}: {Location}", file.Pattern, file.Location); try { @@ -49,7 +51,7 @@ protected override async Task OnFileFoundAsync(ProcessRequest processRequest, ID } catch (Exception e) { - this.Logger.LogFailedReadingFile(file.Location, e); + this.Logger.LogError(e, "Error parsing Podfile.lock at {Location}", file.Location); } } @@ -236,7 +238,7 @@ private void ProcessPodfileLock( } else { - this.Logger.LogWarning($"Missing podspec declaration. podspec={dependency.Podspec}, version={dependency.PodVersion}"); + this.Logger.LogWarning("Missing podspec declaration. podspec={Podspec}, version={PodVersion}", dependency.Podspec, dependency.PodVersion); singleFileComponentRecorder.RegisterPackageParseFailure($"{dependency.Podspec} - {dependency.PodVersion}"); } } diff --git a/src/Microsoft.ComponentDetection.Detectors/dockerfile/DockerfileComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/dockerfile/DockerfileComponentDetector.cs index 0816dca50..c141300f9 100644 --- a/src/Microsoft.ComponentDetection.Detectors/dockerfile/DockerfileComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/dockerfile/DockerfileComponentDetector.cs @@ -1,4 +1,5 @@ namespace Microsoft.ComponentDetection.Detectors.Dockerfile; + using System; using System.Collections.Generic; using System.IO; @@ -9,6 +10,7 @@ using Microsoft.ComponentDetection.Contracts; using Microsoft.ComponentDetection.Contracts.Internal; using Microsoft.ComponentDetection.Contracts.TypedComponent; +using Microsoft.Extensions.Logging; using Valleysoft.DockerfileModel; public class DockerfileComponentDetector : FileComponentDetector, IDefaultOffComponentDetector @@ -21,7 +23,7 @@ public DockerfileComponentDetector( IObservableDirectoryWalkerFactory walkerFactory, ICommandLineInvocationService commandLineInvocationService, IEnvironmentVariableService envVarService, - ILogger logger) + ILogger logger) { this.ComponentStreamEnumerableFactory = componentStreamEnumerableFactory; this.Scanner = walkerFactory; @@ -47,7 +49,7 @@ protected override async Task OnFileFoundAsync(ProcessRequest processRequest, ID var filePath = file.Location; try { - this.Logger.LogInfo($"Discovered dockerfile: {file.Location}"); + this.Logger.LogInformation("Discovered dockerfile: {Location}", file.Location); string contents; using (var reader = new StreamReader(file.Stream)) @@ -60,8 +62,7 @@ protected override async Task OnFileFoundAsync(ProcessRequest processRequest, ID } catch (Exception e) { - this.Logger.LogError($"The file doesn't appear to be a Dockerfile: '{file.Location}'"); - this.Logger.LogException(e, false); + this.Logger.LogError(e, "The file doesn't appear to be a Dockerfile: {Location}", filePath); } } @@ -107,8 +108,7 @@ private DockerReference ProcessDockerfileConstruct(DockerfileConstruct construct } catch (Exception e) { - this.Logger.LogError($"Failed to detect a DockerReference component, the component will not be registered. \n Error Message: <{e.Message}>"); - this.Logger.LogException(e, isError: true, printException: true); + this.Logger.LogError(e, "Failed to detect a DockerReference component, the component will not be registered."); return null; } } diff --git a/src/Microsoft.ComponentDetection.Detectors/go/GoComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/go/GoComponentDetector.cs index 708d5cade..821705508 100644 --- a/src/Microsoft.ComponentDetection.Detectors/go/GoComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/go/GoComponentDetector.cs @@ -1,4 +1,5 @@ namespace Microsoft.ComponentDetection.Detectors.Go; + using System; using System.Collections.Generic; using System.IO; @@ -9,6 +10,7 @@ using Microsoft.ComponentDetection.Contracts; using Microsoft.ComponentDetection.Contracts.Internal; using Microsoft.ComponentDetection.Contracts.TypedComponent; +using Microsoft.Extensions.Logging; using Newtonsoft.Json; public class GoComponentDetector : FileComponentDetector @@ -27,7 +29,7 @@ public GoComponentDetector( IObservableDirectoryWalkerFactory walkerFactory, ICommandLineInvocationService commandLineInvocationService, IEnvironmentVariableService envVarService, - ILogger logger) + ILogger logger) { this.ComponentStreamEnumerableFactory = componentStreamEnumerableFactory; this.Scanner = walkerFactory; @@ -66,14 +68,13 @@ protected override async Task OnFileFoundAsync(ProcessRequest processRequest, ID } else { - this.Logger.LogInfo("Go cli scan was manually disabled, fallback strategy performed." + + this.Logger.LogInformation("Go cli scan was manually disabled, fallback strategy performed." + " More info: https://github.com/microsoft/component-detection/blob/main/docs/detectors/go.md#fallback-detection-strategy"); } } catch (Exception ex) { - this.Logger.LogError($"Failed to detect components using go cli. Location: {file.Location}"); - this.Logger.LogException(ex, isError: true, printException: true); + this.Logger.LogError(ex, "Failed to detect components using go cli. Location: {Location}", file.Location); } finally { @@ -88,14 +89,14 @@ protected override async Task OnFileFoundAsync(ProcessRequest processRequest, ID { case ".MOD": { - this.Logger.LogVerbose("Found Go.mod: " + file.Location); + this.Logger.LogDebug("Found Go.mod: {Location}", file.Location); this.ParseGoModFile(singleFileComponentRecorder, file); break; } case ".SUM": { - this.Logger.LogVerbose("Found Go.sum: " + file.Location); + this.Logger.LogDebug("Found Go.sum: {Location}", file.Location); this.ParseGoSumFile(singleFileComponentRecorder, file); break; } @@ -122,18 +123,18 @@ private async Task UseGoCliToScanAsync(string location, ISingleFileCompone if (!isGoAvailable) { - this.Logger.LogInfo("Go CLI was not found in the system"); + this.Logger.LogInformation("Go CLI was not found in the system"); return false; } - this.Logger.LogInfo("Go CLI was found in system and will be used to generate dependency graph. " + + this.Logger.LogInformation("Go CLI was found in system and will be used to generate dependency graph. " + "Detection time may be improved by activating fallback strategy (https://github.com/microsoft/component-detection/blob/main/docs/detectors/go.md#fallback-detection-strategy). " + "But, it will introduce noise into the detected components."); var goDependenciesProcess = await this.commandLineInvocationService.ExecuteCommandAsync("go", null, workingDirectory: projectRootDirectory, new[] { "list", "-mod=readonly", "-m", "-json", "all" }); if (goDependenciesProcess.ExitCode != 0) { - this.Logger.LogError($"Go CLI command \"go list -m -json all\" failed with error:\n {goDependenciesProcess.StdErr}"); - this.Logger.LogError($"Go CLI could not get dependency build list at location: {location}. Fallback go.sum/go.mod parsing will be used."); + this.Logger.LogError("Go CLI command \"go list -m -json all\" failed with error: {GoDependenciesProcessStdErr}", goDependenciesProcess.StdErr); + this.Logger.LogError("Go CLI could not get dependency build list at location: {Location}. Fallback go.sum/go.mod parsing will be used.", location); return false; } @@ -171,7 +172,7 @@ private void ParseGoModFile( else { var lineTrim = line.Trim(); - this.Logger.LogWarning($"Line could not be parsed for component [{lineTrim}]"); + this.Logger.LogWarning("Line could not be parsed for component [{LineTrim}]", lineTrim); singleFileComponentRecorder.RegisterPackageParseFailure(lineTrim); } } @@ -212,7 +213,7 @@ private void ParseGoSumFile( else { var lineTrim = line.Trim(); - this.Logger.LogWarning($"Line could not be parsed for component [{lineTrim}]"); + this.Logger.LogWarning("Line could not be parsed for component [{LineTrim}]", lineTrim); singleFileComponentRecorder.RegisterPackageParseFailure(lineTrim); } } @@ -250,8 +251,7 @@ private void PopulateDependencyGraph(string goGraphOutput, ISingleFileComponentR continue; } - this.Logger.LogWarning("Unexpected relationship output from go mod graph:"); - this.Logger.LogWarning(relationship); + this.Logger.LogWarning("Unexpected relationship output from go mod graph: {Relationship}", relationship); continue; } @@ -273,7 +273,7 @@ private void PopulateDependencyGraph(string goGraphOutput, ISingleFileComponentR } else { - this.Logger.LogWarning($"Failed to parse components from relationship string {relationship}"); + this.Logger.LogWarning("Failed to parse components from relationship string {Relationship}", relationship); componentRecorder.RegisterPackageParseFailure(relationship); } } diff --git a/src/Microsoft.ComponentDetection.Detectors/gradle/GradleComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/gradle/GradleComponentDetector.cs index 588e0e168..397207a39 100644 --- a/src/Microsoft.ComponentDetection.Detectors/gradle/GradleComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/gradle/GradleComponentDetector.cs @@ -1,4 +1,5 @@ namespace Microsoft.ComponentDetection.Detectors.Gradle; + using System; using System.Collections.Generic; using System.IO; @@ -7,6 +8,7 @@ namespace Microsoft.ComponentDetection.Detectors.Gradle; using Microsoft.ComponentDetection.Contracts; using Microsoft.ComponentDetection.Contracts.Internal; using Microsoft.ComponentDetection.Contracts.TypedComponent; +using Microsoft.Extensions.Logging; public class GradleComponentDetector : FileComponentDetector, IComponentDetector { @@ -15,7 +17,7 @@ public class GradleComponentDetector : FileComponentDetector, IComponentDetector public GradleComponentDetector( IComponentStreamEnumerableFactory componentStreamEnumerableFactory, IObservableDirectoryWalkerFactory walkerFactory, - ILogger logger) + ILogger logger) { this.ComponentStreamEnumerableFactory = componentStreamEnumerableFactory; this.Scanner = walkerFactory; @@ -37,7 +39,7 @@ protected override Task OnFileFoundAsync(ProcessRequest processRequest, IDiction var singleFileComponentRecorder = processRequest.SingleFileComponentRecorder; var file = processRequest.ComponentStream; - this.Logger.LogVerbose("Found Gradle lockfile: " + file.Location); + this.Logger.LogDebug("Found Gradle lockfile: {Location}", file.Location); this.ParseLockfile(singleFileComponentRecorder, file); return Task.CompletedTask; diff --git a/src/Microsoft.ComponentDetection.Detectors/ivy/IvyDetector.cs b/src/Microsoft.ComponentDetection.Detectors/ivy/IvyDetector.cs index 6a173f9d5..a03294d70 100644 --- a/src/Microsoft.ComponentDetection.Detectors/ivy/IvyDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/ivy/IvyDetector.cs @@ -1,4 +1,5 @@ namespace Microsoft.ComponentDetection.Detectors.Ivy; + using System; using System.Collections.Generic; using System.IO; @@ -10,6 +11,7 @@ using Microsoft.ComponentDetection.Contracts.BcdeModels; using Microsoft.ComponentDetection.Contracts.Internal; using Microsoft.ComponentDetection.Contracts.TypedComponent; +using Microsoft.Extensions.Logging; using Newtonsoft.Json.Linq; /// @@ -48,7 +50,7 @@ public IvyDetector( IComponentStreamEnumerableFactory componentStreamEnumerableFactory, IObservableDirectoryWalkerFactory walkerFactory, ICommandLineInvocationService commandLineInvocationService, - ILogger logger) + ILogger logger) { this.ComponentStreamEnumerableFactory = componentStreamEnumerableFactory; this.Scanner = walkerFactory; @@ -73,7 +75,7 @@ protected override async Task> OnPrepareDetectionAsy return processRequests; } - this.Logger.LogVerbose("Skipping Ivy detection as ant is not available in the local PATH."); + this.Logger.LogDebug("Skipping Ivy detection as ant is not available in the local PATH."); return Enumerable.Empty().ToObservable(); } @@ -88,18 +90,18 @@ protected override async Task OnFileFoundAsync(ProcessRequest processRequest, ID { if (File.Exists(ivySettingsFilePath)) { - this.Logger.LogInfo($"Processing {ivyXmlFile.Location} and ivysettings.xml."); + this.Logger.LogInformation("Processing {IvyXmlFileLocation} and ivysettings.xml.", ivyXmlFile.Location); await this.ProcessIvyAndIvySettingsFilesAsync(singleFileComponentRecorder, ivyXmlFile.Location, ivySettingsFilePath); } else { - this.Logger.LogInfo($"Processing {ivyXmlFile.Location}."); + this.Logger.LogInformation("Processing {IvyXmlFile}.", ivyXmlFile.Location); await this.ProcessIvyAndIvySettingsFilesAsync(singleFileComponentRecorder, ivyXmlFile.Location, null); } } else { - this.Logger.LogError($"File {ivyXmlFile.Location} passed to OnFileFound, but does not exist!"); + this.Logger.LogError("File {IvyXmlFileLocation} passed to OnFileFound, but does not exist!", ivyXmlFile.Location); } } @@ -124,7 +126,7 @@ private async Task ProcessIvyAndIvySettingsFilesAsync( try { var workingDirectory = Path.Combine(Path.GetTempPath(), "ComponentDetection_Ivy"); - this.Logger.LogVerbose($"Preparing temporary Ivy project in {workingDirectory}"); + this.Logger.LogDebug("Preparing temporary Ivy project in {WorkingDirectory}", workingDirectory); if (Directory.Exists(workingDirectory)) { Directory.Delete(workingDirectory, recursive: true); @@ -141,10 +143,7 @@ private async Task ProcessIvyAndIvySettingsFilesAsync( } catch (Exception e) { - this.Logger.LogError("Exception occurred during Ivy file processing: " + e); - - // If something went wrong, just ignore the file - this.Logger.LogFailedReadingFile(ivyXmlFile, e); + this.Logger.LogError(e, "Exception occurred processing {FileName} ", ivyXmlFile); } } @@ -183,34 +182,34 @@ private async Task IsAntLocallyAvailableAsync() private async Task RunAntToDetectDependenciesAsync(string workingDirectory) { var ret = false; - this.Logger.LogVerbose($"Executing command `ant resolve-dependencies` in directory {workingDirectory}"); + this.Logger.LogDebug("Executing command `ant resolve-dependencies` in directory {WorkingDirectory}", workingDirectory); var result = await this.commandLineInvocationService.ExecuteCommandAsync(PrimaryCommand, additionalCandidateCommands: AdditionalValidCommands, "-buildfile", workingDirectory, "resolve-dependencies"); if (result.ExitCode == 0) { - this.Logger.LogVerbose("Ant command succeeded"); + this.Logger.LogDebug("Ant command succeeded"); ret = true; } else { - this.Logger.LogError($"Ant command failed with return code {result.ExitCode}"); + this.Logger.LogError("Ant command failed with return code {ExitCode}", result.ExitCode); } if (string.IsNullOrWhiteSpace(result.StdOut)) { - this.Logger.LogVerbose("Ant command wrote nothing to stdout."); + this.Logger.LogDebug("Ant command wrote nothing to stdout."); } else { - this.Logger.LogVerbose("Ant command stdout:\n" + result.StdOut); + this.Logger.LogDebug("Ant command stdout: {AntCmdStdOut}", result.StdOut); } if (string.IsNullOrWhiteSpace(result.StdErr)) { - this.Logger.LogVerbose("Ant command wrote nothing to stderr."); + this.Logger.LogDebug("Ant command wrote nothing to stderr."); } else { - this.Logger.LogWarning("Ant command stderr:\n" + result.StdErr); + this.Logger.LogWarning("Ant command stderr: {AntCmdStdErr}", result.StdErr); } return ret; @@ -236,7 +235,7 @@ private void RegisterUsagesFromFile(ISingleFileComponentRecorder singleFileCompo } else { - this.Logger.LogWarning($"Dependency \"{component.Id}\" could not be resolved by Ivy, and so has not been recorded by Component Detection."); + this.Logger.LogWarning("Dependency \"{MavenComponentId}\" could not be resolved by Ivy, and so has not been recorded by Component Detection.", component.Id); singleFileComponentRecorder.RegisterPackageParseFailure(component.Id); } } diff --git a/src/Microsoft.ComponentDetection.Detectors/linux/LinuxContainerDetector.cs b/src/Microsoft.ComponentDetection.Detectors/linux/LinuxContainerDetector.cs index e174abe65..da6b568da 100644 --- a/src/Microsoft.ComponentDetection.Detectors/linux/LinuxContainerDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/linux/LinuxContainerDetector.cs @@ -1,4 +1,5 @@ namespace Microsoft.ComponentDetection.Detectors.Linux; + using System; using System.Collections.Concurrent; using System.Collections.Generic; @@ -13,14 +14,15 @@ namespace Microsoft.ComponentDetection.Detectors.Linux; using Microsoft.ComponentDetection.Contracts.TypedComponent; using Microsoft.ComponentDetection.Detectors.Linux.Contracts; using Microsoft.ComponentDetection.Detectors.Linux.Exceptions; +using Microsoft.Extensions.Logging; public class LinuxContainerDetector : IComponentDetector { private readonly ILinuxScanner linuxScanner; private readonly IDockerService dockerService; - private readonly ILogger logger; + private readonly ILogger logger; - public LinuxContainerDetector(ILinuxScanner linuxScanner, IDockerService dockerService, ILogger logger) + public LinuxContainerDetector(ILinuxScanner linuxScanner, IDockerService dockerService, ILogger logger) { this.linuxScanner = linuxScanner; this.dockerService = dockerService; @@ -47,7 +49,7 @@ public async Task ExecuteDetectorAsync(ScanRequest if (imagesToProcess == null || !imagesToProcess.Any()) { - this.logger.LogInfo("No instructions received to scan docker images."); + this.logger.LogInformation("No instructions received to scan docker images."); return EmptySuccessfulScan(); } @@ -59,7 +61,7 @@ public async Task ExecuteDetectorAsync(ScanRequest { Os = RuntimeInformation.OSDescription, }; - this.logger.LogInfo("Linux containers are not available on this host."); + this.logger.LogInformation("Linux containers are not available on this host."); return EmptySuccessfulScan(); } @@ -145,7 +147,7 @@ await this.dockerService.TryPullImageAsync(image, cancellationToken))) } catch (Exception e) { - this.logger.LogWarning($"Processing of image {image} failed with exception: {e.Message}"); + this.logger.LogWarning(e, "Processing of image {DockerImage} failed", image); using var record = new LinuxContainerDetectorImageDetectionFailed { ExceptionType = e.GetType().ToString(), @@ -191,7 +193,7 @@ await this.dockerService.TryPullImageAsync(image, cancellationToken))) } catch (Exception e) { - this.logger.LogWarning($"Scanning of image {kvp.Key} failed with exception: {e.Message}"); + this.logger.LogWarning(e, "Scanning of image {KvpKey} failed", kvp.Key); using var record = new LinuxContainerDetectorImageDetectionFailed { ExceptionType = e.GetType().ToString(), @@ -220,14 +222,14 @@ private async Task GetBaseImageLayerCountAsync(ContainerDetails scannedImag if (string.IsNullOrEmpty(scannedImageDetails.BaseImageRef)) { record.BaseImageLayerMessage = $"Base image annotations not found on image {image}, Results will not be mapped to base image layers"; - this.logger.LogInfo(record.BaseImageLayerMessage); + this.logger.LogInformation("Base image annotations not found on image {DockerImage}, Results will not be mapped to base image layers", image); return 0; } if (scannedImageDetails.BaseImageRef == "scratch") { record.BaseImageLayerMessage = $"{image} has no base image"; - this.logger.LogInfo(record.BaseImageLayerMessage); + this.logger.LogInformation("{DockerImage} has no base image", image); return 0; } @@ -240,7 +242,7 @@ private async Task GetBaseImageLayerCountAsync(ContainerDetails scannedImag await this.dockerService.TryPullImageAsync(refWithDigest, cancellationToken))) { record.BaseImageLayerMessage = $"Base image {refWithDigest} could not be found locally and could not be pulled. Results will not be mapped to base image layers"; - this.logger.LogInfo(record.BaseImageLayerMessage); + this.logger.LogInformation("Base image {BaseImage} could not be found locally and could not be pulled. Results will not be mapped to base image layers", refWithDigest); return 0; } @@ -248,7 +250,7 @@ await this.dockerService.TryPullImageAsync(refWithDigest, cancellationToken))) if (!this.ValidateBaseImageLayers(scannedImageDetails, baseImageDetails)) { record.BaseImageLayerMessage = $"Docker image {image} was set to have base image {refWithDigest} but is not built off of it. Results will not be mapped to base image layers"; - this.logger.LogInfo(record.BaseImageLayerMessage); + this.logger.LogInformation("Docker image {DockerImage} was set to have base image {BaseImage} but is not built off of it. Results will not be mapped to base image layers", image, refWithDigest); return 0; } diff --git a/src/Microsoft.ComponentDetection.Detectors/linux/LinuxScanner.cs b/src/Microsoft.ComponentDetection.Detectors/linux/LinuxScanner.cs index c3bc136ae..f903d4132 100644 --- a/src/Microsoft.ComponentDetection.Detectors/linux/LinuxScanner.cs +++ b/src/Microsoft.ComponentDetection.Detectors/linux/LinuxScanner.cs @@ -1,4 +1,5 @@ namespace Microsoft.ComponentDetection.Detectors.Linux; + using System; using System.Collections.Generic; using System.Linq; @@ -9,6 +10,7 @@ namespace Microsoft.ComponentDetection.Detectors.Linux; using Microsoft.ComponentDetection.Contracts.BcdeModels; using Microsoft.ComponentDetection.Contracts.TypedComponent; using Microsoft.ComponentDetection.Detectors.Linux.Contracts; +using Microsoft.Extensions.Logging; using Newtonsoft.Json; public class LinuxScanner : ILinuxScanner @@ -27,9 +29,9 @@ public class LinuxScanner : ILinuxScanner private static readonly int SemaphoreTimeout = Convert.ToInt32(TimeSpan.FromHours(1).TotalMilliseconds); private readonly IDockerService dockerService; - private readonly ILogger logger; + private readonly ILogger logger; - public LinuxScanner(IDockerService dockerService, ILogger logger) + public LinuxScanner(IDockerService dockerService, ILogger logger) { this.dockerService = dockerService; this.logger = logger; @@ -62,14 +64,14 @@ public async Task> ScanLinuxAsync(string catch (Exception e) { syftTelemetryRecord.Exception = JsonConvert.SerializeObject(e); - this.logger.LogException(e, false); + this.logger.LogError(e, "Failed to run syft"); throw; } } else { record.SemaphoreFailure = true; - this.logger.LogWarning($"Failed to enter the docker semaphore for image {imageHash}"); + this.logger.LogWarning("Failed to enter the docker semaphore for image {ImageHash}", imageHash); } } finally diff --git a/src/Microsoft.ComponentDetection.Detectors/maven/MavenCommandService.cs b/src/Microsoft.ComponentDetection.Detectors/maven/MavenCommandService.cs index ab28e6654..abf46d4d5 100644 --- a/src/Microsoft.ComponentDetection.Detectors/maven/MavenCommandService.cs +++ b/src/Microsoft.ComponentDetection.Detectors/maven/MavenCommandService.cs @@ -1,9 +1,11 @@ -namespace Microsoft.ComponentDetection.Detectors.Maven; +namespace Microsoft.ComponentDetection.Detectors.Maven; + using System; using System.IO; using System.Threading.Tasks; using Microsoft.ComponentDetection.Contracts; using Microsoft.ComponentDetection.Contracts.Internal; +using Microsoft.Extensions.Logging; public class MavenCommandService : IMavenCommandService { @@ -15,12 +17,12 @@ public class MavenCommandService : IMavenCommandService private readonly ICommandLineInvocationService commandLineInvocationService; private readonly IMavenStyleDependencyGraphParserService parserService; - private readonly ILogger logger; + private readonly ILogger logger; public MavenCommandService( ICommandLineInvocationService commandLineInvocationService, IMavenStyleDependencyGraphParserService parserService, - ILogger logger) + ILogger logger) { this.commandLineInvocationService = commandLineInvocationService; this.parserService = parserService; @@ -41,8 +43,8 @@ public async Task GenerateDependenciesFileAsync(ProcessRequest processRequest) var result = await this.commandLineInvocationService.ExecuteCommandAsync(PrimaryCommand, AdditionalValidCommands, cliParameters); if (result.ExitCode != 0) { - this.logger.LogVerbose($"Mvn execution failed for pom file: {pomFile.Location}"); - this.logger.LogError(string.IsNullOrEmpty(result.StdErr) ? result.StdOut : result.StdErr); + this.logger.LogDebug("Mvn execution failed for pom file: {PomFileLocation}", pomFile.Location); + this.logger.LogError("Mvn output: {MvnStdErr}", string.IsNullOrEmpty(result.StdErr) ? result.StdOut : result.StdErr); processRequest.SingleFileComponentRecorder.RegisterPackageParseFailure(pomFile.Location); } } diff --git a/src/Microsoft.ComponentDetection.Detectors/maven/MvnCliComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/maven/MvnCliComponentDetector.cs index 396b5477d..2f70232aa 100644 --- a/src/Microsoft.ComponentDetection.Detectors/maven/MvnCliComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/maven/MvnCliComponentDetector.cs @@ -1,4 +1,5 @@ namespace Microsoft.ComponentDetection.Detectors.Maven; + using System; using System.Collections.Generic; using System.IO; @@ -11,6 +12,7 @@ namespace Microsoft.ComponentDetection.Detectors.Maven; using Microsoft.ComponentDetection.Contracts; using Microsoft.ComponentDetection.Contracts.Internal; using Microsoft.ComponentDetection.Contracts.TypedComponent; +using Microsoft.Extensions.Logging; public class MvnCliComponentDetector : FileComponentDetector { @@ -20,7 +22,7 @@ public MvnCliComponentDetector( IComponentStreamEnumerableFactory componentStreamEnumerableFactory, IObservableDirectoryWalkerFactory walkerFactory, IMavenCommandService mavenCommandService, - ILogger logger) + ILogger logger) { this.ComponentStreamEnumerableFactory = componentStreamEnumerableFactory; this.Scanner = walkerFactory; @@ -42,7 +44,7 @@ protected override async Task> OnPrepareDetectionAsy { if (!await this.mavenCommandService.MavenCLIExistsAsync()) { - this.Logger.LogVerbose("Skipping maven detection as maven is not available in the local PATH."); + this.Logger.LogDebug("Skipping maven detection as maven is not available in the local PATH."); return Enumerable.Empty().ToObservable(); } @@ -143,7 +145,7 @@ private IObservable RemoveNestedPomXmls(IObservable string.Equals(Path.GetFileName(x.Location), "pom.xml", StringComparison.OrdinalIgnoreCase)) != null) { - this.Logger.LogVerbose($"Ignoring pom.xml at {item.Location}, as it has a parent pom.xml that will be processed at {current.Name}\\pom.xml ."); + this.Logger.LogDebug("Ignoring pom.xml at {ChildPomXmlLocation}, as it has a parent pom.xml that will be processed at {ParentDirName}\\pom.xml .", item.Location, current.Name); break; } diff --git a/src/Microsoft.ComponentDetection.Detectors/npm/NpmComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/npm/NpmComponentDetector.cs index 57c877376..ef1bf63db 100644 --- a/src/Microsoft.ComponentDetection.Detectors/npm/NpmComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/npm/NpmComponentDetector.cs @@ -9,6 +9,7 @@ using Microsoft.ComponentDetection.Contracts; using Microsoft.ComponentDetection.Contracts.Internal; using Microsoft.ComponentDetection.Contracts.TypedComponent; +using Microsoft.Extensions.Logging; using Newtonsoft.Json.Linq; public class NpmComponentDetector : FileComponentDetector @@ -16,7 +17,7 @@ public class NpmComponentDetector : FileComponentDetector public NpmComponentDetector( IComponentStreamEnumerableFactory componentStreamEnumerableFactory, IObservableDirectoryWalkerFactory walkerFactory, - ILogger logger) + ILogger logger) { this.ComponentStreamEnumerableFactory = componentStreamEnumerableFactory; this.Scanner = walkerFactory; @@ -55,8 +56,7 @@ await this.SafeProcessAllPackageJTokensAsync(filePath, contents, (token) => { if (token["name"] == null || token["version"] == null) { - this.Logger.LogInfo($"{filePath} does not contain a name and/or version. These are required fields for a valid package.json file." + - $"It and its dependencies will not be registered."); + this.Logger.LogInformation("{BadPackageJson} does not contain a name and/or version. These are required fields for a valid package.json file. It and its dependencies will not be registered.", filePath); return false; } @@ -79,7 +79,7 @@ protected virtual bool ProcessIndividualPackageJTokens(string filePath, ISingleF if (!SemanticVersion.TryParse(version, out _)) { - this.Logger.LogWarning($"Unable to parse version \"{version}\" for package \"{name}\" found at path \"{filePath}\". This may indicate an invalid npm package component and it will not be registered."); + this.Logger.LogWarning("Unable to parse version {NpmPackageVersion} for package {NpmPackageName} found at path {NpmPackageLocation}. This may indicate an invalid npm package component and it will not be registered.", version, name, filePath); singleFileComponentRecorder.RegisterPackageParseFailure($"{name} - {version}"); return false; } @@ -99,9 +99,7 @@ private async Task SafeProcessAllPackageJTokensAsync(string sourceFilePath, stri catch (Exception e) { // If something went wrong, just ignore the component - this.Logger.LogInfo($"Could not parse Jtokens from file {sourceFilePath}."); - this.Logger.LogFailedReadingFile(sourceFilePath, e); - return; + this.Logger.LogInformation(e, "Could not parse Jtokens from file {PackageJsonFilePaths}.", sourceFilePath); } } @@ -144,13 +142,13 @@ private NpmAuthor GetAuthor(JToken authorToken, string packageName, string fileP } else { - this.Logger.LogWarning($"Unable to parse author:[{authorString}] for package:[{packageName}] found at path:[{filePath}]. This may indicate an invalid npm package author, and author will not be registered."); + this.Logger.LogWarning("Unable to parse author:[{NpmAuthorString}] for package:[{NpmPackageName}] found at path:[{NpmPackageLocation}]. This may indicate an invalid npm package author, and author will not be registered.", authorString, packageName, filePath); return null; } if (string.IsNullOrEmpty(authorName)) { - this.Logger.LogWarning($"Unable to parse author:[{authorString}] for package:[{packageName}] found at path:[{filePath}]. This may indicate an invalid npm package author, and author will not be registered."); + this.Logger.LogWarning("Unable to parse author:[{NpmAuthorString}] for package:[{NpmPackageName}] found at path:[{NpmPackageLocation}]. This may indicate an invalid npm package author, and author will not be registered.", authorString, packageName, filePath); return null; } diff --git a/src/Microsoft.ComponentDetection.Detectors/npm/NpmComponentDetectorWithRoots.cs b/src/Microsoft.ComponentDetection.Detectors/npm/NpmComponentDetectorWithRoots.cs index 990047910..d3e219cf6 100644 --- a/src/Microsoft.ComponentDetection.Detectors/npm/NpmComponentDetectorWithRoots.cs +++ b/src/Microsoft.ComponentDetection.Detectors/npm/NpmComponentDetectorWithRoots.cs @@ -1,4 +1,5 @@ namespace Microsoft.ComponentDetection.Detectors.Npm; + using System; using System.Collections.Generic; using System.IO; @@ -9,6 +10,7 @@ using Microsoft.ComponentDetection.Contracts; using Microsoft.ComponentDetection.Contracts.Internal; using Microsoft.ComponentDetection.Contracts.TypedComponent; +using Microsoft.Extensions.Logging; using Newtonsoft.Json; using Newtonsoft.Json.Linq; @@ -29,7 +31,7 @@ public NpmComponentDetectorWithRoots( IComponentStreamEnumerableFactory componentStreamEnumerableFactory, IObservableDirectoryWalkerFactory walkerFactory, IPathUtilityService pathUtilityService, - ILogger logger) + ILogger logger) { this.ComponentStreamEnumerableFactory = componentStreamEnumerableFactory; this.Scanner = walkerFactory; @@ -111,8 +113,7 @@ await this.SafeProcessAllPackageJTokensAsync(file, (token) => { if (!foundUnderLerna && (token["name"] == null || token["version"] == null || string.IsNullOrWhiteSpace(token["name"].Value()) || string.IsNullOrWhiteSpace(token["version"].Value()))) { - this.Logger.LogInfo($"{file.Location} does not contain a valid name and/or version. These are required fields for a valid package-lock.json file." + - $"It and its dependencies will not be registered."); + this.Logger.LogInformation("{PackageLogJsonFile} does not contain a valid name and/or version. These are required fields for a valid package-lock.json file. It and its dependencies will not be registered.", file.Location); return false; } @@ -132,8 +133,7 @@ protected Task ProcessAllPackageJTokensAsync(IComponentStream componentStream, J } catch (Exception ex) { - this.Logger.LogInfo($"Could not read {componentStream.Location} file."); - this.Logger.LogFailedReadingFile(componentStream.Location, ex); + this.Logger.LogInformation(ex, "Could not read {ComponentStreamFile} file.", componentStream.Location); return Task.CompletedTask; } @@ -210,7 +210,7 @@ private IObservable RemoveNodeModuleNestedFiles(IObservable logger) { this.ComponentStreamEnumerableFactory = componentStreamEnumerableFactory; this.Scanner = walkerFactory; @@ -72,8 +73,8 @@ private async Task ProcessAdditionalDirectoryAsync(ProcessRequest processRequest // Only paths outside of our sourceDirectory need to be added if (!rootPath.IsBaseOf(new Uri(additionalPath.FullName + Path.DirectorySeparatorChar))) { - this.Logger.LogInfo($"Found path override in nuget configuration file. Adding {additionalPath} to the package search path."); - this.Logger.LogWarning($"Path {additionalPath} is not rooted in the source tree. More components may be detected than expected if this path is shared across code projects."); + this.Logger.LogInformation("Found path override in nuget configuration file. Adding {NuGetAdditionalPath} to the package search path.", additionalPath); + this.Logger.LogWarning("Path {NuGetAdditionalPath} is not rooted in the source tree. More components may be detected than expected if this path is shared across code projects.", additionalPath); this.Scanner.Initialize(additionalPath, (name, directoryName) => false, 1); @@ -125,7 +126,7 @@ private async Task ProcessFileAsync(ProcessRequest processRequest) if (!NuGetVersion.TryParse(version, out var parsedVer)) { - this.Logger.LogInfo($"Version '{version}' from {stream.Location} could not be parsed as a NuGet version"); + this.Logger.LogInformation("Version '{NuspecVersion}' from {NuspecLocation} could not be parsed as a NuGet version", version, stream.Location); singleFileComponentRecorder.RegisterPackageParseFailure(stream.Location); return; @@ -140,7 +141,7 @@ private async Task ProcessFileAsync(ProcessRequest processRequest) catch (Exception e) { // If something went wrong, just ignore the component - this.Logger.LogFailedReadingFile(stream.Location, e); + this.Logger.LogError(e, "Error parsing NuGet component from {NuspecLocation}", stream.Location); singleFileComponentRecorder.RegisterPackageParseFailure(stream.Location); } } @@ -216,7 +217,7 @@ private IList GetRepositoryPathsFromNugetConfig(IComponentStream } else { - this.Logger.LogWarning($"Excluding discovered path {potentialPath} from location {componentStream.Location} as it could not be determined to be valid."); + this.Logger.LogWarning("Excluding discovered path {PotentialPath} from location {ComponentStreamLocation} as it could not be determined to be valid.", potentialPath, componentStream.Location); continue; } diff --git a/src/Microsoft.ComponentDetection.Detectors/nuget/NuGetPackagesConfigDetector.cs b/src/Microsoft.ComponentDetection.Detectors/nuget/NuGetPackagesConfigDetector.cs index 68e5be5a8..48e67867e 100644 --- a/src/Microsoft.ComponentDetection.Detectors/nuget/NuGetPackagesConfigDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/nuget/NuGetPackagesConfigDetector.cs @@ -8,13 +8,14 @@ namespace Microsoft.ComponentDetection.Detectors.NuGet; using Microsoft.ComponentDetection.Contracts; using Microsoft.ComponentDetection.Contracts.Internal; using Microsoft.ComponentDetection.Contracts.TypedComponent; +using Microsoft.Extensions.Logging; public class NuGetPackagesConfigDetector : FileComponentDetector { public NuGetPackagesConfigDetector( IComponentStreamEnumerableFactory componentStreamEnumerableFactory, IObservableDirectoryWalkerFactory walkerFactory, - ILogger logger) + ILogger logger) { this.ComponentStreamEnumerableFactory = componentStreamEnumerableFactory; this.Scanner = walkerFactory; @@ -51,7 +52,7 @@ protected override Task OnFileFoundAsync(ProcessRequest processRequest, IDiction } catch (Exception e) when (e is PackagesConfigReaderException or XmlException) { - this.Logger.LogFailedReadingFile(processRequest.ComponentStream.Location, e); + this.Logger.LogError(e, "Failed to read packages.config file {File}", processRequest.ComponentStream.Location); } return Task.CompletedTask; diff --git a/src/Microsoft.ComponentDetection.Detectors/nuget/NuGetProjectModelProjectCentricComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/nuget/NuGetProjectModelProjectCentricComponentDetector.cs index ec1388eb2..1b893cb20 100644 --- a/src/Microsoft.ComponentDetection.Detectors/nuget/NuGetProjectModelProjectCentricComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/nuget/NuGetProjectModelProjectCentricComponentDetector.cs @@ -12,6 +12,7 @@ namespace Microsoft.ComponentDetection.Detectors.NuGet; using Microsoft.ComponentDetection.Contracts; using Microsoft.ComponentDetection.Contracts.Internal; using Microsoft.ComponentDetection.Contracts.TypedComponent; +using Microsoft.Extensions.Logging; using Newtonsoft.Json; public class NuGetProjectModelProjectCentricComponentDetector : FileComponentDetector @@ -24,8 +25,6 @@ public class NuGetProjectModelProjectCentricComponentDetector : FileComponentDet private readonly List netCoreFrameworkNames = new List { "Microsoft.AspNetCore.App", "Microsoft.AspNetCore.Razor.Design", "Microsoft.NETCore.App" }; - private readonly HashSet alreadyLoggedWarnings = new HashSet(); - // This list is meant to encompass all net standard dependencies, but likely contains some net core app 1.x ones, too. // The specific guidance we got around populating this list is to do so based on creating a dotnet core 1.x app to make sure we had the complete // set of netstandard.library files that could show up in later sdk versions. @@ -191,7 +190,7 @@ public NuGetProjectModelProjectCentricComponentDetector( IComponentStreamEnumerableFactory componentStreamEnumerableFactory, IObservableDirectoryWalkerFactory walkerFactory, IFileUtilityService fileUtilityService, - ILogger logger) + ILogger logger) { this.ComponentStreamEnumerableFactory = componentStreamEnumerableFactory; this.Scanner = walkerFactory; @@ -243,7 +242,7 @@ protected override Task OnFileFoundAsync(ProcessRequest processRequest, IDiction catch (Exception e) { // If something went wrong, just ignore the package - this.Logger.LogFailedReadingFile(processRequest.ComponentStream.Location, e); + this.Logger.LogError(e, "Failed to process NuGet lockfile {NuGetLockFile}", processRequest.ComponentStream.Location); } return Task.CompletedTask; @@ -382,12 +381,12 @@ private LockFileLibrary GetLibraryComponentWithDependencyLookup(IList logger; private bool checkedMaxEntriesVariable; @@ -62,7 +63,7 @@ public class PyPiClient : IPyPiClient FinalCacheSize = 0, }; - public PyPiClient(IEnvironmentVariableService environmentVariableService, ILogger logger) + public PyPiClient(IEnvironmentVariableService environmentVariableService, ILogger logger) { this.environmentVariableService = environmentVariableService; this.cacheTelemetry = new PypiCacheTelemetryRecord @@ -90,7 +91,7 @@ public async Task> FetchPackageDependenciesAsy if (!response.IsSuccessStatusCode) { - this.logger.LogWarning($"Http GET at {release.Url} failed with status code {response.StatusCode}"); + this.logger.LogWarning("Http GET at {ReleaseUrl} failed with status code {ResponseStatusCode}", release.Url, response.StatusCode); return dependencies; } @@ -154,7 +155,13 @@ public async Task>> GetRele { using var r = new PypiRetryTelemetryRecord { Name = spec.Name, DependencySpecifiers = spec.DependencySpecifiers?.ToArray(), StatusCode = result.Result.StatusCode }; - this.logger.LogWarning($"Received {(int)result.Result.StatusCode} {result.Result.ReasonPhrase} from {requestUri}. Waiting {timeSpan} before retry attempt {retryCount}"); + this.logger.LogWarning( + "Received {StatusCode} {ReasonPhrase} from {RequestUri}. Waiting {TimeSpan} before retry attempt {RetryCount}", + result.Result.StatusCode, + result.Result.ReasonPhrase, + requestUri, + timeSpan, + retryCount); Interlocked.Increment(ref this.retries); }) @@ -181,7 +188,7 @@ public async Task>> GetRele { using var r = new PypiFailureTelemetryRecord { Name = spec.Name, DependencySpecifiers = spec.DependencySpecifiers?.ToArray(), StatusCode = request.StatusCode }; - this.logger.LogWarning($"Received {(int)request.StatusCode} {request.ReasonPhrase} from {requestUri}"); + this.logger.LogWarning("Received {StatusCode} {ReasonPhrase} from {RequestUri}", request.StatusCode, request.ReasonPhrase, requestUri); return new SortedDictionary>(); } @@ -204,8 +211,12 @@ public async Task>> GetRele } catch (ArgumentException ae) { - this.logger.LogError($"Component {release.Key} : {JsonConvert.SerializeObject(release.Value)} could not be added to the sorted list of pip components for spec={spec.Name}. Usually this happens with unexpected PyPi version formats (e.g. prerelease/dev versions). Error details follow:"); - this.logger.LogException(ae, true); + this.logger.LogError( + ae, + "Component {ReleaseKey} : {ReleaseValue)} could not be added to the sorted list of pip components for spec={SpecName}. Usually this happens with unexpected PyPi version formats (e.g. prerelease/dev versions).", + release.Key, + JsonConvert.SerializeObject(release.Value), + spec.Name); continue; } } @@ -229,11 +240,11 @@ private async Task GetAndCachePyPiResponseAsync(Uri uri) if (this.cachedResponses.TryGetValue(uri, out HttpResponseMessage result)) { this.cacheTelemetry.NumCacheHits++; - this.logger.LogVerbose("Retrieved cached Python data from " + uri); + this.logger.LogDebug("Retrieved cached Python data from {Uri}", uri); return result; } - this.logger.LogInfo("Getting Python data from " + uri); + this.logger.LogInformation("Getting Python data from {Uri}", uri); var response = await HttpClient.GetAsync(uri); // The `first - wins` response accepted into the cache. This might be different from the input if another caller wins the race. @@ -254,7 +265,7 @@ private void InitializeNonDefaultMemoryCache() var maxEntriesVariable = this.environmentVariableService.GetEnvironmentVariable("PyPiMaxCacheEntries"); if (!string.IsNullOrEmpty(maxEntriesVariable) && long.TryParse(maxEntriesVariable, out var maxEntries)) { - this.logger.LogInfo($"Setting IPyPiClient max cache entries to {maxEntries}"); + this.logger.LogInformation("Setting IPyPiClient max cache entries to {MaxEntries}", maxEntries); this.cachedResponses = new MemoryCache(new MemoryCacheOptions { SizeLimit = maxEntries }); } diff --git a/src/Microsoft.ComponentDetection.Detectors/pip/PipComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/pip/PipComponentDetector.cs index e760f788c..75a203179 100644 --- a/src/Microsoft.ComponentDetection.Detectors/pip/PipComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/pip/PipComponentDetector.cs @@ -1,4 +1,5 @@ namespace Microsoft.ComponentDetection.Detectors.Pip; + using System; using System.Collections.Generic; using System.Linq; @@ -7,6 +8,7 @@ namespace Microsoft.ComponentDetection.Detectors.Pip; using Microsoft.ComponentDetection.Contracts; using Microsoft.ComponentDetection.Contracts.Internal; using Microsoft.ComponentDetection.Contracts.TypedComponent; +using Microsoft.Extensions.Logging; public class PipComponentDetector : FileComponentDetector { @@ -18,7 +20,7 @@ public PipComponentDetector( IObservableDirectoryWalkerFactory walkerFactory, IPythonCommandService pythonCommandService, IPythonResolver pythonResolver, - ILogger logger) + ILogger logger) { this.ComponentStreamEnumerableFactory = componentStreamEnumerableFactory; this.Scanner = walkerFactory; @@ -42,7 +44,7 @@ protected override async Task> OnPrepareDetectionAsy this.CurrentScanRequest.DetectorArgs.TryGetValue("Pip.PythonExePath", out var pythonExePath); if (!await this.pythonCommandService.PythonExistsAsync(pythonExePath)) { - this.Logger.LogInfo($"No python found on system. Python detection will not run."); + this.Logger.LogInformation($"No python found on system. Python detection will not run."); return Enumerable.Empty().ToObservable(); } @@ -79,7 +81,7 @@ protected override async Task OnFileFoundAsync(ProcessRequest processRequest, ID } catch (Exception e) { - this.Logger.LogFailedReadingFile(file.Location, e); + this.Logger.LogError(e, "Error while parsing pip components in {File}", file.Location); } } diff --git a/src/Microsoft.ComponentDetection.Detectors/pip/PythonResolver.cs b/src/Microsoft.ComponentDetection.Detectors/pip/PythonResolver.cs index ea4d4c16e..823ebebaa 100644 --- a/src/Microsoft.ComponentDetection.Detectors/pip/PythonResolver.cs +++ b/src/Microsoft.ComponentDetection.Detectors/pip/PythonResolver.cs @@ -1,17 +1,19 @@ namespace Microsoft.ComponentDetection.Detectors.Pip; + using System; using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; using Microsoft.ComponentDetection.Contracts; using Microsoft.ComponentDetection.Contracts.TypedComponent; +using Microsoft.Extensions.Logging; public class PythonResolver : IPythonResolver { private readonly IPyPiClient pypiClient; - private readonly ILogger logger; + private readonly ILogger logger; - public PythonResolver(IPyPiClient pypiClient, ILogger logger) + public PythonResolver(IPyPiClient pypiClient, ILogger logger) { this.pypiClient = pypiClient; this.logger = logger; @@ -53,7 +55,9 @@ public async Task> ResolveRootsAsync(ISingleFileComponentRec } else { - this.logger.LogWarning($"Root dependency {rootPackage.Name} not found on pypi. Skipping package."); + this.logger.LogWarning( + "Root dependency {RootPackageName} not found on pypi. Skipping package.", + rootPackage.Name); singleFileComponentRecorder.RegisterPackageParseFailure(rootPackage.Name); } } @@ -83,13 +87,15 @@ private async Task> ProcessQueueAsync(ISingleFileComponentRe } else if (node != null) { - this.logger.LogWarning($"Candidate version ({node.Value.Id}) for {dependencyNode.Name} already exists in map and the version is NOT valid."); - this.logger.LogWarning($"Specifiers: {string.Join(',', dependencyNode.DependencySpecifiers)} for package {currentNode.Name} caused this."); + this.logger.LogWarning("Candidate version ({NodeValueId}) for {DependencyName} already exists in map and the version is NOT valid.", node.Value.Id, dependencyNode.Name); + this.logger.LogWarning("Specifiers: {DependencySpecifiers} for package {CurrentNodeName} caused this.", string.Join(',', dependencyNode.DependencySpecifiers), currentNode.Name); // The currently selected version is invalid, try to see if there is another valid version available if (!await this.InvalidateAndReprocessAsync(state, node, dependencyNode)) { - this.logger.LogWarning($"Version Resolution for {dependencyNode.Name} failed, assuming last valid version is used."); + this.logger.LogWarning( + "Version Resolution for {DependencyName} failed, assuming last valid version is used.", + dependencyNode.Name); // there is no valid version available for the node, dependencies are incompatible, } @@ -111,6 +117,9 @@ private async Task> ProcessQueueAsync(ISingleFileComponentRe } else { + this.logger.LogWarning( + "Dependency Package {DependencyName} not found in Pypi. Skipping package", + dependencyNode.Name); singleFileComponentRecorder.RegisterPackageParseFailure(dependencyNode.Name); } } diff --git a/src/Microsoft.ComponentDetection.Detectors/pnpm/PnpmComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/pnpm/PnpmComponentDetector.cs index b531935ee..7cbd6a1e4 100644 --- a/src/Microsoft.ComponentDetection.Detectors/pnpm/PnpmComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/pnpm/PnpmComponentDetector.cs @@ -1,4 +1,5 @@ namespace Microsoft.ComponentDetection.Detectors.Pnpm; + using System; using System.Collections.Generic; using System.Linq; @@ -6,13 +7,14 @@ namespace Microsoft.ComponentDetection.Detectors.Pnpm; using Microsoft.ComponentDetection.Contracts; using Microsoft.ComponentDetection.Contracts.Internal; using Microsoft.ComponentDetection.Contracts.TypedComponent; +using Microsoft.Extensions.Logging; public class PnpmComponentDetector : FileComponentDetector { public PnpmComponentDetector( IComponentStreamEnumerableFactory componentStreamEnumerableFactory, IObservableDirectoryWalkerFactory walkerFactory, - ILogger logger) + ILogger logger) { this.ComponentStreamEnumerableFactory = componentStreamEnumerableFactory; this.Scanner = walkerFactory; @@ -38,11 +40,11 @@ protected override async Task OnFileFoundAsync(ProcessRequest processRequest, ID var singleFileComponentRecorder = processRequest.SingleFileComponentRecorder; var file = processRequest.ComponentStream; - this.Logger.LogVerbose("Found yaml file: " + file.Location); + this.Logger.LogDebug("Found yaml file: {YamlFile}", file.Location); var skippedFolder = this.SkippedFolders.FirstOrDefault(folder => file.Location.Contains(folder)); if (!string.IsNullOrEmpty(skippedFolder)) { - this.Logger.LogVerbose($"Skipping found file, it was detected as being within a {skippedFolder} folder."); + this.Logger.LogDebug("Skipping found file, it was detected as being within a {SkippedFolder} folder.", skippedFolder); } try @@ -52,7 +54,7 @@ protected override async Task OnFileFoundAsync(ProcessRequest processRequest, ID } catch (Exception e) { - this.Logger.LogFailedReadingFile(file.Location, e); + this.Logger.LogError(e, "Failed to read pnpm yaml file {File}", file.Location); } } diff --git a/src/Microsoft.ComponentDetection.Detectors/poetry/PoetryComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/poetry/PoetryComponentDetector.cs index d2e4b9177..779ccd778 100644 --- a/src/Microsoft.ComponentDetection.Detectors/poetry/PoetryComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/poetry/PoetryComponentDetector.cs @@ -1,4 +1,5 @@ namespace Microsoft.ComponentDetection.Detectors.Poetry; + using System; using System.Collections.Generic; using System.IO; @@ -8,6 +9,7 @@ namespace Microsoft.ComponentDetection.Detectors.Poetry; using Microsoft.ComponentDetection.Contracts.Internal; using Microsoft.ComponentDetection.Contracts.TypedComponent; using Microsoft.ComponentDetection.Detectors.Poetry.Contracts; +using Microsoft.Extensions.Logging; using Tomlyn; public class PoetryComponentDetector : FileComponentDetector, IExperimentalDetector @@ -15,7 +17,7 @@ public class PoetryComponentDetector : FileComponentDetector, IExperimentalDetec public PoetryComponentDetector( IComponentStreamEnumerableFactory componentStreamEnumerableFactory, IObservableDirectoryWalkerFactory walkerFactory, - ILogger logger) + ILogger logger) { this.ComponentStreamEnumerableFactory = componentStreamEnumerableFactory; this.Scanner = walkerFactory; @@ -36,7 +38,7 @@ protected override async Task OnFileFoundAsync(ProcessRequest processRequest, ID { var singleFileComponentRecorder = processRequest.SingleFileComponentRecorder; var poetryLockFile = processRequest.ComponentStream; - this.Logger.LogVerbose("Found Poetry lockfile: " + poetryLockFile); + this.Logger.LogDebug("Found Poetry lockfile {PoetryLockFile}", poetryLockFile); var reader = new StreamReader(poetryLockFile.Stream); var options = new TomlModelOptions diff --git a/src/Microsoft.ComponentDetection.Detectors/ruby/RubyComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/ruby/RubyComponentDetector.cs index 110a56aea..dba7f9f92 100644 --- a/src/Microsoft.ComponentDetection.Detectors/ruby/RubyComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/ruby/RubyComponentDetector.cs @@ -26,6 +26,7 @@ // is necessary to investigate if this section is a new adition or always has been there. namespace Microsoft.ComponentDetection.Detectors.Ruby; + using System; using System.Collections.Generic; using System.IO; @@ -34,6 +35,7 @@ namespace Microsoft.ComponentDetection.Detectors.Ruby; using Microsoft.ComponentDetection.Contracts; using Microsoft.ComponentDetection.Contracts.Internal; using Microsoft.ComponentDetection.Contracts.TypedComponent; +using Microsoft.Extensions.Logging; public class RubyComponentDetector : FileComponentDetector { @@ -44,7 +46,7 @@ public class RubyComponentDetector : FileComponentDetector public RubyComponentDetector( IComponentStreamEnumerableFactory componentStreamEnumerableFactory, IObservableDirectoryWalkerFactory walkerFactory, - ILogger logger) + ILogger logger) { this.ComponentStreamEnumerableFactory = componentStreamEnumerableFactory; this.Scanner = walkerFactory; @@ -74,7 +76,7 @@ protected override Task OnFileFoundAsync(ProcessRequest processRequest, IDiction var singleFileComponentRecorder = processRequest.SingleFileComponentRecorder; var file = processRequest.ComponentStream; - this.Logger.LogVerbose("Found Gemfile.lock: " + file.Location); + this.Logger.LogDebug("Found Gemfile.lock {FileLocation}", file.Location); this.ParseGemLockFile(singleFileComponentRecorder, file); return Task.CompletedTask; @@ -142,8 +144,8 @@ private void ParseGemLockFile(ISingleFileComponentRecorder singleFileComponentRe else { // Throw this line away. Is this malformed? We were expecting a header - this.Logger.LogVerbose(lines[0]); - this.Logger.LogVerbose("Appears to be malformed/is not expected here. Expected heading."); + this.Logger.LogDebug("{MalformedLine}", lines[0]); + this.Logger.LogDebug("Appears to be malformed/is not expected here. Expected heading. {Line}", lines[0]); lines.RemoveAt(0); } } @@ -220,7 +222,7 @@ private void ParseSection(ISingleFileComponentRecorder singleFileComponentRecord if (this.IsVersionRelative(version)) { - this.Logger.LogWarning($"Found component with invalid version, name = {name} and version = {version}"); + this.Logger.LogWarning("Found component with invalid version, name = {RubyComponentName} and version = {RubyComponentVersion}", name, version); singleFileComponentRecorder.RegisterPackageParseFailure($"{name} - {version}"); wasParentDependencyExcluded = true; continue; diff --git a/src/Microsoft.ComponentDetection.Detectors/rust/RustCrateDetector.cs b/src/Microsoft.ComponentDetection.Detectors/rust/RustCrateDetector.cs index f16c43104..2d0304068 100644 --- a/src/Microsoft.ComponentDetection.Detectors/rust/RustCrateDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/rust/RustCrateDetector.cs @@ -1,4 +1,5 @@ namespace Microsoft.ComponentDetection.Detectors.Rust; + using System; using System.Collections.Generic; using System.IO; @@ -10,6 +11,7 @@ using Microsoft.ComponentDetection.Contracts.Internal; using Microsoft.ComponentDetection.Contracts.TypedComponent; using Microsoft.ComponentDetection.Detectors.Rust.Contracts; +using Microsoft.Extensions.Logging; using Tomlyn; public class RustCrateDetector : FileComponentDetector @@ -24,7 +26,7 @@ public class RustCrateDetector : FileComponentDetector public RustCrateDetector( IComponentStreamEnumerableFactory componentStreamEnumerableFactory, IObservableDirectoryWalkerFactory walkerFactory, - ILogger logger) + ILogger logger) { this.ComponentStreamEnumerableFactory = componentStreamEnumerableFactory; this.Scanner = walkerFactory; @@ -147,7 +149,7 @@ protected override async Task OnFileFoundAsync(ProcessRequest processRequest, ID catch (Exception e) { // If something went wrong, just ignore the file - this.Logger.LogFailedReadingFile(cargoLockFile.Location, e); + this.Logger.LogError(e, "Failed to process Cargo.lock file '{CargoLockLocation}'", cargoLockFile.Location); } await Task.CompletedTask; @@ -240,7 +242,7 @@ private void ProcessDependency( record.PackageInfo = $"{parentPackage.Name}, {parentPackage.Version}, {parentPackage.Source}"; record.Dependencies = dependency; - this.Logger.LogFailedReadingFile(cargoLockFile.Location, e); + this.Logger.LogError(e, "Failed to process Cargo.lock file '{CargoLockLocation}'", cargoLockFile.Location); singleFileComponentRecorder.RegisterPackageParseFailure(record.PackageInfo); } } diff --git a/src/Microsoft.ComponentDetection.Detectors/spdx/Spdx22ComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/spdx/Spdx22ComponentDetector.cs index f74c6832d..fe710910d 100644 --- a/src/Microsoft.ComponentDetection.Detectors/spdx/Spdx22ComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/spdx/Spdx22ComponentDetector.cs @@ -1,4 +1,5 @@ namespace Microsoft.ComponentDetection.Detectors.Spdx; + using System; using System.Collections.Generic; using System.IO; @@ -8,6 +9,7 @@ using Microsoft.ComponentDetection.Contracts; using Microsoft.ComponentDetection.Contracts.Internal; using Microsoft.ComponentDetection.Contracts.TypedComponent; +using Microsoft.Extensions.Logging; using Newtonsoft.Json; using Newtonsoft.Json.Linq; @@ -22,7 +24,7 @@ public class Spdx22ComponentDetector : FileComponentDetector, IDefaultOffCompone public Spdx22ComponentDetector( IComponentStreamEnumerableFactory componentStreamEnumerableFactory, IObservableDirectoryWalkerFactory walkerFactory, - ILogger logger) + ILogger logger) { this.ComponentStreamEnumerableFactory = componentStreamEnumerableFactory; this.Scanner = walkerFactory; @@ -42,7 +44,7 @@ public Spdx22ComponentDetector( protected override Task OnFileFoundAsync(ProcessRequest processRequest, IDictionary detectorArgs) { - this.Logger.LogVerbose($"Discovered SPDX2.2 manifest file at: {processRequest.ComponentStream.Location}"); + this.Logger.LogDebug("Discovered SPDX2.2 manifest file at: {ManifestLocation}", processRequest.ComponentStream.Location); var file = processRequest.ComponentStream; try @@ -68,22 +70,22 @@ protected override Task OnFileFoundAsync(ProcessRequest processRequest, IDiction } else { - this.Logger.LogWarning($"Discovered SPDX at {processRequest.ComponentStream.Location} is not SPDX-2.2 document, skipping"); + this.Logger.LogWarning("Discovered SPDX at {ManifestLocation} is not SPDX-2.2 document, skipping", processRequest.ComponentStream.Location); } } else { - this.Logger.LogWarning($"Discovered SPDX file at {processRequest.ComponentStream.Location} is not a valid document, skipping"); + this.Logger.LogWarning("Discovered SPDX file at {ManifestLocation} is not a valid document, skipping", processRequest.ComponentStream.Location); } } catch (JsonReaderException) { - this.Logger.LogWarning($"Unable to parse file at {processRequest.ComponentStream.Location}, skipping"); + this.Logger.LogWarning("Unable to parse file at {ManifestLocation}, skipping", processRequest.ComponentStream.Location); } } catch (Exception e) { - this.Logger.LogFailedReadingFile(file.Location, e); + this.Logger.LogError(e, "Error while processing SPDX file at {ManifestLocation}", processRequest.ComponentStream.Location); } return Task.CompletedTask; @@ -100,12 +102,12 @@ private SpdxComponent ConvertJObjectToSbomComponent(ProcessRequest processReques if (rootElements?.Length > 1) { - this.Logger.LogWarning($"SPDX file at {processRequest.ComponentStream.Location} has more than one element in documentDescribes, first will be selected as root element."); + this.Logger.LogWarning("SPDX file at {ManifestLocation} has more than one element in documentDescribes, first will be selected as root element.", processRequest.ComponentStream.Location); } if (rootElements != null && !rootElements.Any()) { - this.Logger.LogWarning($"SPDX file at {processRequest.ComponentStream.Location} does not have root elements in documentDescribes section, considering SPDXRef-Document as a root element."); + this.Logger.LogWarning("SPDX file at {ManifestLocation} does not have root elements in documentDescribes section, considering SPDXRef-Document as a root element.", processRequest.ComponentStream.Location); } var rootElementId = rootElements?.FirstOrDefault() ?? "SPDXRef-Document"; diff --git a/src/Microsoft.ComponentDetection.Detectors/vcpkg/VcpkgComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/vcpkg/VcpkgComponentDetector.cs index da6fbde2b..660ba91e6 100644 --- a/src/Microsoft.ComponentDetection.Detectors/vcpkg/VcpkgComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/vcpkg/VcpkgComponentDetector.cs @@ -1,4 +1,5 @@ namespace Microsoft.ComponentDetection.Detectors.Vcpkg; + using System; using System.Collections.Generic; using System.IO; @@ -8,6 +9,7 @@ using Microsoft.ComponentDetection.Contracts.Internal; using Microsoft.ComponentDetection.Contracts.TypedComponent; using Microsoft.ComponentDetection.Detectors.Vcpkg.Contracts; +using Microsoft.Extensions.Logging; using Newtonsoft.Json; public class VcpkgComponentDetector : FileComponentDetector, IExperimentalDetector @@ -22,7 +24,7 @@ public VcpkgComponentDetector( IObservableDirectoryWalkerFactory walkerFactory, ICommandLineInvocationService commandLineInvocationService, IEnvironmentVariableService environmentVariableService, - ILogger logger) + ILogger logger) { this.ComponentStreamEnumerableFactory = componentStreamEnumerableFactory; this.Scanner = walkerFactory; @@ -46,7 +48,7 @@ protected override async Task OnFileFoundAsync(ProcessRequest processRequest, ID var singleFileComponentRecorder = processRequest.SingleFileComponentRecorder; var file = processRequest.ComponentStream; - this.Logger.LogVerbose($"vcpkg detector found {file}"); + this.Logger.LogDebug("vcpkg detector found {File}", file); var projectRootDirectory = Directory.GetParent(file.Location); if (this.projectRoots.Any(path => projectRootDirectory.FullName.StartsWith(path))) @@ -86,7 +88,7 @@ private async Task ParseSpdxFileAsync( continue; } - this.Logger.LogVerbose($"vcpkg parsed package {item.Name}"); + this.Logger.LogDebug("vcpkg parsed package {PackageName}", item.Name); if (item.SPDXID == "SPDXRef-port") { var split = item.VersionInfo.Split('#'); @@ -113,9 +115,9 @@ private async Task ParseSpdxFileAsync( singleFileComponentRecorder.RegisterUsage(new DetectedComponent(component)); } } - catch (Exception) + catch (Exception e) { - this.Logger.LogWarning($"failed while handling {item.Name}"); + this.Logger.LogWarning(e, "failed while handling {ItemName}", item.Name); singleFileComponentRecorder.RegisterPackageParseFailure(item.Name); } } diff --git a/src/Microsoft.ComponentDetection.Detectors/yarn/IYarnLockFileFactory.cs b/src/Microsoft.ComponentDetection.Detectors/yarn/IYarnLockFileFactory.cs index 593dc7fd0..350e47e3e 100644 --- a/src/Microsoft.ComponentDetection.Detectors/yarn/IYarnLockFileFactory.cs +++ b/src/Microsoft.ComponentDetection.Detectors/yarn/IYarnLockFileFactory.cs @@ -3,6 +3,7 @@ namespace Microsoft.ComponentDetection.Detectors.Yarn; using System.IO; using System.Threading.Tasks; using Microsoft.ComponentDetection.Contracts; +using Microsoft.Extensions.Logging; public interface IYarnLockFileFactory { diff --git a/src/Microsoft.ComponentDetection.Detectors/yarn/IYarnLockParser.cs b/src/Microsoft.ComponentDetection.Detectors/yarn/IYarnLockParser.cs index 6336058ef..a8b67fcfd 100644 --- a/src/Microsoft.ComponentDetection.Detectors/yarn/IYarnLockParser.cs +++ b/src/Microsoft.ComponentDetection.Detectors/yarn/IYarnLockParser.cs @@ -1,6 +1,8 @@ namespace Microsoft.ComponentDetection.Detectors.Yarn; + using Microsoft.ComponentDetection.Contracts; using Microsoft.ComponentDetection.Detectors.Yarn.Parsers; +using Microsoft.Extensions.Logging; public interface IYarnLockParser { diff --git a/src/Microsoft.ComponentDetection.Detectors/yarn/Parsers/YarnLockParser.cs b/src/Microsoft.ComponentDetection.Detectors/yarn/Parsers/YarnLockParser.cs index a0358a63e..53940916e 100644 --- a/src/Microsoft.ComponentDetection.Detectors/yarn/Parsers/YarnLockParser.cs +++ b/src/Microsoft.ComponentDetection.Detectors/yarn/Parsers/YarnLockParser.cs @@ -1,8 +1,10 @@ namespace Microsoft.ComponentDetection.Detectors.Yarn.Parsers; + using System; using System.Collections.Generic; using System.Linq; using Microsoft.ComponentDetection.Contracts; +using Microsoft.Extensions.Logging; public class YarnLockParser : IYarnLockParser { @@ -16,9 +18,9 @@ public class YarnLockParser : IYarnLockParser private static readonly List SupportedVersions = new List { YarnLockVersion.V1, YarnLockVersion.V2 }; - private readonly ILogger logger; + private readonly ILogger logger; - public YarnLockParser(ILogger logger) => this.logger = logger; + public YarnLockParser(ILogger logger) => this.logger = logger; public static string NormalizeVersion(string version) { @@ -63,13 +65,13 @@ public YarnLockFile Parse(ISingleFileComponentRecorder singleFileComponentRecord if (string.IsNullOrWhiteSpace(yarnEntry.Name)) { - logger.LogWarning($"Failed to read a name for block {block.Title}. The entry will be skipped."); + logger.LogWarning("Failed to read a name for block {BlockTitle}. The entry will be skipped.", block.Title); continue; } if (!block.Values.TryGetValue(VersionString, out var version)) { - logger.LogWarning($"Failed to read a version for {yarnEntry.Name}. The entry will be skipped."); + logger.LogWarning("Failed to read a version for {YarnEntryName}. The entry will be skipped.", yarnEntry.Name); singleFileComponentRecorder.RegisterPackageParseFailure(yarnEntry.Name); continue; } diff --git a/src/Microsoft.ComponentDetection.Detectors/yarn/YarnLockComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/yarn/YarnLockComponentDetector.cs index b269e0c2b..d52c2bf16 100644 --- a/src/Microsoft.ComponentDetection.Detectors/yarn/YarnLockComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/yarn/YarnLockComponentDetector.cs @@ -1,4 +1,5 @@ namespace Microsoft.ComponentDetection.Detectors.Yarn; + using System; using System.Collections.Generic; using System.IO; @@ -10,6 +11,7 @@ using Microsoft.ComponentDetection.Contracts.Internal; using Microsoft.ComponentDetection.Contracts.TypedComponent; using Microsoft.ComponentDetection.Detectors.Npm; +using Microsoft.Extensions.Logging; public class YarnLockComponentDetector : FileComponentDetector { @@ -19,7 +21,7 @@ public YarnLockComponentDetector( IComponentStreamEnumerableFactory componentStreamEnumerableFactory, IObservableDirectoryWalkerFactory walkerFactory, IYarnLockFileFactory yarnLockFileFactory, - ILogger logger) + ILogger logger) { this.yarnLockFileFactory = yarnLockFileFactory; this.ComponentStreamEnumerableFactory = componentStreamEnumerableFactory; @@ -49,11 +51,11 @@ protected override async Task OnFileFoundAsync(ProcessRequest processRequest, ID var skippedFolder = this.SkippedFolders.FirstOrDefault(folder => file.Location.Contains(folder)); if (!string.IsNullOrEmpty(skippedFolder)) { - this.Logger.LogInfo($"Yarn.Lock file {file.Location} was found in a {skippedFolder} folder and will be skipped."); + this.Logger.LogInformation("Yarn.Lock file {YarnLockLocation} was found in a {SkippedFolder} folder and will be skipped.", file.Location, skippedFolder); return; } - this.Logger.LogInfo($"Processing file {file.Location}"); + this.Logger.LogInformation("Processing file {YarnLockLocation}", file.Location); try { @@ -62,8 +64,7 @@ protected override async Task OnFileFoundAsync(ProcessRequest processRequest, ID } catch (Exception ex) { - this.Logger.LogBuildWarning($"Could not read components from file {file.Location}."); - this.Logger.LogFailedReadingFile(file.Location, ex); + this.Logger.LogError(ex, "Could not read components from file {YarnLockLocation}.", file.Location); } } @@ -84,7 +85,7 @@ private void DetectComponents(YarnLockFile file, string location, ISingleFileCom var addSuccessful = yarnPackages.TryAdd(key, entry); if (!addSuccessful) { - this.Logger.LogWarning($"Found duplicate entry {key} in {location}"); + this.Logger.LogWarning("Found duplicate entry {Key} in {Location}", key, location); } } } @@ -167,7 +168,7 @@ private void ParseTreeWithAssignedRoot(YarnEntry root, Dictionary yarnWorkspaces, DirectoryInf foreach (var stream in componentStreams) { - this.Logger.LogInfo($"{stream.Location} found for workspace {workspacePattern}"); + this.Logger.LogInformation("{ComponentLocation} found for workspace {WorkspacePattern}", stream.Location, workspacePattern); var combinedDependencies = NpmComponentUtilities.TryGetAllPackageJsonDependencies(stream.Stream, out _); foreach (var dependency in combinedDependencies) diff --git a/src/Microsoft.ComponentDetection.Detectors/yarn/YarnLockFileFactory.cs b/src/Microsoft.ComponentDetection.Detectors/yarn/YarnLockFileFactory.cs index 0d19a4ab7..763399ef6 100644 --- a/src/Microsoft.ComponentDetection.Detectors/yarn/YarnLockFileFactory.cs +++ b/src/Microsoft.ComponentDetection.Detectors/yarn/YarnLockFileFactory.cs @@ -1,9 +1,11 @@ namespace Microsoft.ComponentDetection.Detectors.Yarn; + using System.Collections.Generic; using System.IO; using System.Threading.Tasks; using Microsoft.ComponentDetection.Contracts; using Microsoft.ComponentDetection.Detectors.Yarn.Parsers; +using Microsoft.Extensions.Logging; public class YarnLockFileFactory : IYarnLockFileFactory { diff --git a/src/Microsoft.ComponentDetection.Orchestrator/Extensions/ServiceCollectionExtensions.cs b/src/Microsoft.ComponentDetection.Orchestrator/Extensions/ServiceCollectionExtensions.cs index e9640aaf2..f031e8866 100644 --- a/src/Microsoft.ComponentDetection.Orchestrator/Extensions/ServiceCollectionExtensions.cs +++ b/src/Microsoft.ComponentDetection.Orchestrator/Extensions/ServiceCollectionExtensions.cs @@ -25,6 +25,8 @@ using Microsoft.ComponentDetection.Orchestrator.Services; using Microsoft.ComponentDetection.Orchestrator.Services.GraphTranslation; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Serilog.Extensions.Logging; public static class ServiceCollectionExtensions { @@ -48,7 +50,6 @@ public static IServiceCollection AddComponentDetection(this IServiceCollection s services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); - services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); @@ -129,4 +130,25 @@ public static IServiceCollection AddComponentDetection(this IServiceCollection s return services; } + + public static IServiceCollection ConfigureLoggingProviders(this IServiceCollection services) + { + var providers = new LoggerProviderCollection(); + services.AddSingleton(providers); + services.AddSingleton(sc => + { + var providerCollection = sc.GetService(); + var factory = new SerilogLoggerFactory(null, true, providerCollection); + + foreach (var provider in sc.GetServices()) + { + factory.AddProvider(provider); + } + + return factory; + }); + services.AddLogging(l => l.AddFilter(null, LogLevel.Trace)); + + return services; + } } diff --git a/src/Microsoft.ComponentDetection.Orchestrator/Microsoft.ComponentDetection.Orchestrator.csproj b/src/Microsoft.ComponentDetection.Orchestrator/Microsoft.ComponentDetection.Orchestrator.csproj index 471e8dd85..387ffd99c 100644 --- a/src/Microsoft.ComponentDetection.Orchestrator/Microsoft.ComponentDetection.Orchestrator.csproj +++ b/src/Microsoft.ComponentDetection.Orchestrator/Microsoft.ComponentDetection.Orchestrator.csproj @@ -4,8 +4,14 @@ + + + + + + @@ -15,4 +21,4 @@ - \ No newline at end of file + diff --git a/src/Microsoft.ComponentDetection.Orchestrator/Orchestrator.cs b/src/Microsoft.ComponentDetection.Orchestrator/Orchestrator.cs index 3ad7df685..6c00debae 100644 --- a/src/Microsoft.ComponentDetection.Orchestrator/Orchestrator.cs +++ b/src/Microsoft.ComponentDetection.Orchestrator/Orchestrator.cs @@ -2,6 +2,7 @@ namespace Microsoft.ComponentDetection.Orchestrator; using System; using System.Collections.Generic; using System.Diagnostics; +using System.IO; using System.Linq; using System.Reflection; using System.Runtime.InteropServices; @@ -18,7 +19,12 @@ namespace Microsoft.ComponentDetection.Orchestrator; using Microsoft.ComponentDetection.Orchestrator.ArgumentSets; using Microsoft.ComponentDetection.Orchestrator.Services; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; using Newtonsoft.Json; +using Serilog; +using Serilog.Events; +using Serilog.Extensions.Hosting; +using Serilog.Extensions.Logging; public class Orchestrator { @@ -28,14 +34,14 @@ public class Orchestrator private readonly IEnumerable argumentHandlers; private readonly IFileWritingService fileWritingService; private readonly IArgumentHelper argumentHelper; - private readonly ILogger logger; + private readonly ILogger logger; public Orchestrator( IServiceProvider serviceProvider, IEnumerable argumentHandlers, IFileWritingService fileWritingService, IArgumentHelper argumentHelper, - ILogger logger) + ILogger logger) { this.serviceProvider = serviceProvider; this.argumentHandlers = argumentHandlers; @@ -55,6 +61,27 @@ public async Task LoadAsync(string[] args, CancellationToken cancell baseArguments = new BaseArguments(); } + var logFile = Path.Combine( + baseArguments.Output ?? Path.GetTempPath(), + $"GovCompDisc_Log{DateTime.Now:yyyyMMddHHmmssfff}.log"); + + var reloadableLogger = (ReloadableLogger)Log.Logger; + reloadableLogger.Reload(configuration => + configuration + .WriteTo.Console() + .WriteTo.File(logFile, buffered: true) + .WriteTo.Providers(this.serviceProvider.GetRequiredService()) + .MinimumLevel.Is(baseArguments.Verbosity switch + { + VerbosityMode.Quiet => LogEventLevel.Error, + VerbosityMode.Normal => LogEventLevel.Information, + VerbosityMode.Verbose => LogEventLevel.Debug, + _ => throw new ArgumentOutOfRangeException(nameof(baseArguments.Verbosity), "Invalid verbosity level"), + }) + .Enrich.FromLogContext()); + + this.logger.LogInformation("Log file: {LogFile}", logFile); + // This is required so TelemetryRelay can be accessed via it's static singleton // It should be refactored out at a later date TelemetryRelay.Instance.Init(this.serviceProvider.GetRequiredService>()); @@ -81,7 +108,7 @@ public async Task LoadAsync(string[] args, CancellationToken cancell // The order of these things is a little weird, but done this way mostly to prevent any of the logic inside if blocks from being duplicated if (shouldFailureBeSuppressed) { - this.logger.LogInfo("The scan had some detections complete while others encountered errors. The log file should indicate any issues that happened during the scan."); + this.logger.LogInformation("The scan had some detections complete while others encountered errors. The log file should indicate any issues that happened during the scan."); } if (returnResult.ResultCode == ProcessingResultCode.TimeoutError) @@ -131,8 +158,8 @@ await parsedArguments.WithParsedAsync(async argumentSet => telemetryRecord.Arguments = JsonConvert.SerializeObject(argumentSet); this.fileWritingService.Init(argumentSet.Output); - this.logger.Init(argumentSet.Verbosity, writeLinePrefix: true); - this.logger.LogInfo($"Run correlation id: {TelemetryConstants.CorrelationId}"); + + this.logger.LogInformation("Run correlation id: {CorrelationId}", TelemetryConstants.CorrelationId); return await this.DispatchAsync(argumentSet, cancellationToken); }); @@ -226,7 +253,7 @@ private async Task DispatchAsync(IScanArguments arguments, Cancellat } catch (TimeoutException timeoutException) { - this.logger.LogError(timeoutException.Message); + this.logger.LogError(timeoutException, "The scan timed out."); scanResult.ResultCode = ProcessingResultCode.TimeoutError; } @@ -254,8 +281,7 @@ private async Task SafelyExecuteAsync(BcdeExecutionTelemetryRecord r var e = ae.GetBaseException(); if (e is InvalidUserInputException) { - this.logger.LogError($"Something bad happened, is everything configured correctly?"); - this.logger.LogException(e, isError: true, printException: true); + this.logger.LogError(e, "Something bad happened, is everything configured correctly?"); record.ErrorMessage = e.ToString(); result.ResultCode = ProcessingResultCode.InputError; @@ -265,8 +291,7 @@ private async Task SafelyExecuteAsync(BcdeExecutionTelemetryRecord r else { // On an exception, return error to dotnet core - this.logger.LogError($"There was an unexpected error: "); - this.logger.LogException(e, isError: true); + this.logger.LogError(e, "There was an unexpected error"); var errorMessage = new StringBuilder(); errorMessage.AppendLine(e.ToString()); @@ -274,9 +299,8 @@ private async Task SafelyExecuteAsync(BcdeExecutionTelemetryRecord r { foreach (var loaderException in refEx.LoaderExceptions) { - var loaderExceptionString = loaderException.ToString(); - this.logger.LogError(loaderExceptionString); - errorMessage.AppendLine(loaderExceptionString); + this.logger.LogError(loaderException, "Got exception"); + errorMessage.AppendLine(loaderException.ToString()); } } diff --git a/src/Microsoft.ComponentDetection.Orchestrator/Services/BcdeDevCommandService.cs b/src/Microsoft.ComponentDetection.Orchestrator/Services/BcdeDevCommandService.cs index 387607288..5d08eb0fb 100644 --- a/src/Microsoft.ComponentDetection.Orchestrator/Services/BcdeDevCommandService.cs +++ b/src/Microsoft.ComponentDetection.Orchestrator/Services/BcdeDevCommandService.cs @@ -1,20 +1,21 @@ -namespace Microsoft.ComponentDetection.Orchestrator.Services; +namespace Microsoft.ComponentDetection.Orchestrator.Services; using System; using System.Linq; using System.Threading.Tasks; -using Microsoft.ComponentDetection.Contracts; using Microsoft.ComponentDetection.Contracts.BcdeModels; using Microsoft.ComponentDetection.Orchestrator.ArgumentSets; +using Microsoft.Extensions.Logging; -public class BcdeDevCommandService : ServiceBase, IArgumentHandlingService +public class BcdeDevCommandService : IArgumentHandlingService { private readonly IBcdeScanExecutionService bcdeScanExecutionService; + private readonly ILogger logger; - public BcdeDevCommandService(IBcdeScanExecutionService bcdeScanExecutionService, ILogger logger) + public BcdeDevCommandService(IBcdeScanExecutionService bcdeScanExecutionService, ILogger logger) { this.bcdeScanExecutionService = bcdeScanExecutionService; - this.Logger = logger; + this.logger = logger; } public bool CanHandle(IScanArguments arguments) diff --git a/src/Microsoft.ComponentDetection.Orchestrator/Services/BcdeScanCommandService.cs b/src/Microsoft.ComponentDetection.Orchestrator/Services/BcdeScanCommandService.cs index d93f9dd17..7bfc48944 100644 --- a/src/Microsoft.ComponentDetection.Orchestrator/Services/BcdeScanCommandService.cs +++ b/src/Microsoft.ComponentDetection.Orchestrator/Services/BcdeScanCommandService.cs @@ -1,28 +1,29 @@ -namespace Microsoft.ComponentDetection.Orchestrator.Services; +namespace Microsoft.ComponentDetection.Orchestrator.Services; using System.IO; using System.Threading.Tasks; using Microsoft.ComponentDetection.Common; -using Microsoft.ComponentDetection.Contracts; using Microsoft.ComponentDetection.Contracts.BcdeModels; using Microsoft.ComponentDetection.Orchestrator.ArgumentSets; +using Microsoft.Extensions.Logging; using Newtonsoft.Json; -public class BcdeScanCommandService : ServiceBase, IArgumentHandlingService +public class BcdeScanCommandService : IArgumentHandlingService { public const string ManifestRelativePath = "ScanManifest_{timestamp}.json"; private readonly IFileWritingService fileWritingService; private readonly IBcdeScanExecutionService bcdeScanExecutionService; + private readonly ILogger logger; public BcdeScanCommandService( IFileWritingService fileWritingService, IBcdeScanExecutionService bcdeScanExecutionService, - ILogger logger) + ILogger logger) { this.fileWritingService = fileWritingService; this.bcdeScanExecutionService = bcdeScanExecutionService; - this.Logger = logger; + this.logger = logger; } public bool CanHandle(IScanArguments arguments) @@ -44,12 +45,12 @@ private void WriteComponentManifest(IDetectionArguments detectionArguments, Scan if (detectionArguments.ManifestFile != null) { - this.Logger.LogInfo($"Scan Manifest file: {detectionArguments.ManifestFile.FullName}"); + this.logger.LogInformation("Scan Manifest file: {ManifestFile}", detectionArguments.ManifestFile.FullName); userRequestedManifestPath = detectionArguments.ManifestFile; } else { - this.Logger.LogInfo($"Scan Manifest file: {this.fileWritingService.ResolveFilePath(ManifestRelativePath)}"); + this.logger.LogInformation("Scan Manifest file: {ManifestFile}", this.fileWritingService.ResolveFilePath(ManifestRelativePath)); } if (userRequestedManifestPath == null) diff --git a/src/Microsoft.ComponentDetection.Orchestrator/Services/BcdeScanExecutionService.cs b/src/Microsoft.ComponentDetection.Orchestrator/Services/BcdeScanExecutionService.cs index 1b4ef63a7..01f5c7109 100644 --- a/src/Microsoft.ComponentDetection.Orchestrator/Services/BcdeScanExecutionService.cs +++ b/src/Microsoft.ComponentDetection.Orchestrator/Services/BcdeScanExecutionService.cs @@ -8,36 +8,38 @@ using Microsoft.ComponentDetection.Contracts.BcdeModels; using Microsoft.ComponentDetection.Orchestrator.ArgumentSets; using Microsoft.ComponentDetection.Orchestrator.Services.GraphTranslation; +using Microsoft.Extensions.Logging; -public class BcdeScanExecutionService : ServiceBase, IBcdeScanExecutionService +public class BcdeScanExecutionService : IBcdeScanExecutionService { private readonly IEnumerable detectors; private readonly IDetectorProcessingService detectorProcessingService; private readonly IDetectorRestrictionService detectorRestrictionService; private readonly IGraphTranslationService graphTranslationService; + private readonly ILogger logger; public BcdeScanExecutionService( IEnumerable detectors, IDetectorProcessingService detectorProcessingService, IDetectorRestrictionService detectorRestrictionService, IGraphTranslationService graphTranslationService, - ILogger logger) + ILogger logger) { this.detectors = detectors; this.detectorProcessingService = detectorProcessingService; this.detectorRestrictionService = detectorRestrictionService; this.graphTranslationService = graphTranslationService; - this.Logger = logger; + this.logger = logger; } public async Task ExecuteScanAsync(IDetectionArguments detectionArguments) { - this.Logger.LogCreateLoggingGroup(); + using var scope = this.logger.BeginScope("Executing BCDE scan"); var detectorRestrictions = this.GetDetectorRestrictions(detectionArguments); var detectors = this.detectorRestrictionService.ApplyRestrictions(detectorRestrictions, this.detectors).ToImmutableList(); - this.Logger.LogVerbose($"Finished applying restrictions to detectors."); + this.logger.LogDebug("Finished applying restrictions to detectors."); var processingResult = await this.detectorProcessingService.ProcessDetectorsAsync(detectionArguments, detectors, detectorRestrictions); var scanResult = this.graphTranslationService.GenerateScanResultFromProcessingResult(processingResult, detectionArguments); diff --git a/src/Microsoft.ComponentDetection.Orchestrator/Services/DetectorListingCommandService.cs b/src/Microsoft.ComponentDetection.Orchestrator/Services/DetectorListingCommandService.cs index ef699ebc7..93e3627c6 100644 --- a/src/Microsoft.ComponentDetection.Orchestrator/Services/DetectorListingCommandService.cs +++ b/src/Microsoft.ComponentDetection.Orchestrator/Services/DetectorListingCommandService.cs @@ -1,20 +1,22 @@ -namespace Microsoft.ComponentDetection.Orchestrator.Services; +namespace Microsoft.ComponentDetection.Orchestrator.Services; using System.Collections.Generic; using System.Threading.Tasks; using Microsoft.ComponentDetection.Contracts; using Microsoft.ComponentDetection.Contracts.BcdeModels; using Microsoft.ComponentDetection.Orchestrator.ArgumentSets; +using Microsoft.Extensions.Logging; -public class DetectorListingCommandService : ServiceBase, IArgumentHandlingService +public class DetectorListingCommandService : IArgumentHandlingService { private readonly IEnumerable detectors; + private readonly ILogger logger; public DetectorListingCommandService( IEnumerable detectors, - ILogger logger) + ILogger logger) { this.detectors = detectors; - this.Logger = logger; + this.logger = logger; } public bool CanHandle(IScanArguments arguments) @@ -33,9 +35,11 @@ public async Task HandleAsync(IScanArguments arguments) private async Task ListDetectorsAsync(IScanArguments listArguments) { + this.logger.LogInformation("Detectors:"); + foreach (var detector in this.detectors) { - this.Logger.LogInfo($"{detector.Id}"); + this.logger.LogInformation("{DetectorId}", detector.Id); } return await Task.FromResult(ProcessingResultCode.Success); diff --git a/src/Microsoft.ComponentDetection.Orchestrator/Services/DetectorProcessingService.cs b/src/Microsoft.ComponentDetection.Orchestrator/Services/DetectorProcessingService.cs index 86aa96293..c32d5585b 100644 --- a/src/Microsoft.ComponentDetection.Orchestrator/Services/DetectorProcessingService.cs +++ b/src/Microsoft.ComponentDetection.Orchestrator/Services/DetectorProcessingService.cs @@ -14,25 +14,27 @@ using Microsoft.ComponentDetection.Contracts; using Microsoft.ComponentDetection.Contracts.BcdeModels; using Microsoft.ComponentDetection.Orchestrator.ArgumentSets; +using Microsoft.Extensions.Logging; using Newtonsoft.Json; using static System.Environment; -public class DetectorProcessingService : ServiceBase, IDetectorProcessingService +public class DetectorProcessingService : IDetectorProcessingService { private readonly IObservableDirectoryWalkerFactory scanner; + private readonly ILogger logger; public DetectorProcessingService( IObservableDirectoryWalkerFactory scanner, - ILogger logger) + ILogger logger) { this.scanner = scanner; - this.Logger = logger; + this.logger = logger; } public async Task ProcessDetectorsAsync(IDetectionArguments detectionArguments, IEnumerable detectors, DetectorRestrictions detectorRestrictions) { - this.Logger.LogCreateLoggingGroup(); - this.Logger.LogInfo($"Finding components..."); + using var scope = this.logger.BeginScope("Processing detectors"); + this.logger.LogInformation($"Finding components..."); var stopwatch = Stopwatch.StartNew(); var exitCode = ProcessingResultCode.Success; @@ -51,7 +53,7 @@ public async Task ProcessDetectorsAsync(IDetectionArgu var providerStopwatch = new Stopwatch(); providerStopwatch.Start(); - var componentRecorder = new ComponentRecorder(this.Logger, !detector.NeedsAutomaticRootDependencyCalculation); + var componentRecorder = new ComponentRecorder(this.logger, !detector.NeedsAutomaticRootDependencyCalculation); var isExperimentalDetector = detector is IExperimentalDetector && !(detectorRestrictions.ExplicitlyEnabledDetectorIds?.Contains(detector.Id)).GetValueOrDefault(); @@ -62,7 +64,7 @@ public async Task ProcessDetectorsAsync(IDetectionArgu using (var record = new DetectorExecutionTelemetryRecord()) { result = await this.WithExperimentalScanGuardsAsync( - () => detector.ExecuteDetectorAsync(new ScanRequest(detectionArguments.SourceDirectory, exclusionPredicate, this.Logger, detectorArguments, detectionArguments.DockerImagesToScan, componentRecorder)), + () => detector.ExecuteDetectorAsync(new ScanRequest(detectionArguments.SourceDirectory, exclusionPredicate, this.logger, detectorArguments, detectionArguments.DockerImagesToScan, componentRecorder)), isExperimentalDetector, record); @@ -117,7 +119,7 @@ public async Task ProcessDetectorsAsync(IDetectionArgu var detectorProcessingResult = this.ConvertDetectorResultsIntoResult(results, exitCode); var totalElapsedTime = stopwatch.Elapsed.TotalSeconds; - this.LogTabularOutput(this.Logger, providerElapsedTime, totalElapsedTime); + this.LogTabularOutput(this.logger, providerElapsedTime, totalElapsedTime); // If there are components which are skipped due to connection or parsing // errors, log them by detector. @@ -132,22 +134,22 @@ public async Task ProcessDetectorsAsync(IDetectionArgu if (!parseWarningShown) { - this.Logger.LogCreateLoggingGroup(); - this.Logger.LogWarning($"Some components or files were not detected due to parsing failures or connectivity issues."); - this.Logger.LogWarning($"Please review the logs above for more detailed information."); + using var parseWarningScope = this.logger.BeginScope("Parse warnings"); + this.logger.LogWarning("Some components or files were not detected due to parsing failures or connectivity issues."); + this.logger.LogWarning("Please review the logs above for more detailed information."); parseWarningShown = true; } - this.Logger.LogCreateLoggingGroup(); - this.Logger.LogWarning($"Components skipped for {detector.Id} detector:"); + using var scGroup = this.logger.BeginScope("Skipped Components"); + this.logger.LogWarning("Components skipped for {DetectorId} detector:", detector.Id); foreach (var component in skippedComponents) { - this.Logger.LogWarning($"- {component}"); + this.logger.LogWarning("- {Component}", component); } } - this.Logger.LogCreateLoggingGroup(); - this.Logger.LogInfo($"Detection time: {totalElapsedTime} seconds."); + using var dtScope = this.logger.BeginScope("Detection Time"); + this.logger.LogInformation("Detection time: {DetectionTime} seconds.", totalElapsedTime); return detectorProcessingResult; } @@ -190,7 +192,7 @@ public ExcludeDirectoryPredicate GenerateDirectoryExclusionPredicate(string orig && (pathOfParentOfDirectoryToConsider.Equals(pathOfParentOfDirectoryToExclude, StringComparison.Ordinal) || pathOfParentOfDirectoryToConsider.ToString().Equals(valueTuple.rootedLinuxSymlinkCompatibleRelativePathToExclude, StringComparison.Ordinal))) { - this.Logger.LogVerbose($"Excluding folder {Path.Combine(pathOfParentOfDirectoryToConsider.ToString(), nameOfDirectoryToConsider.ToString())}."); + this.logger.LogDebug("Excluding folder {Folder}.", Path.Combine(pathOfParentOfDirectoryToConsider, nameOfDirectoryToConsider)); return true; } } @@ -222,7 +224,7 @@ public ExcludeDirectoryPredicate GenerateDirectoryExclusionPredicate(string orig { if (minimatcherKeyValue.Value.IsMatch(path)) { - this.Logger.LogVerbose($"Excluding folder {path} because it matched glob {minimatcherKeyValue.Key}."); + this.logger.LogDebug("Excluding folder {Path} because it matched glob {Glob}.", path, minimatcherKeyValue.Key); return true; } @@ -336,7 +338,7 @@ private void LogTabularOutput(ILogger logger, ConcurrentDictionary oldDetectorIds = new List { "MSLicenseDevNpm", "MSLicenseDevNpmList", "MSLicenseNpm", "MSLicenseNpmList" }; private readonly string newDetectorId = "NpmWithRoots"; - private readonly ILogger logger; + private readonly ILogger logger; - public DetectorRestrictionService(ILogger logger) => this.logger = logger; + public DetectorRestrictionService(ILogger logger) => this.logger = logger; public IEnumerable ApplyRestrictions(DetectorRestrictions restrictions, IEnumerable detectors) { @@ -46,7 +47,7 @@ public IEnumerable ApplyRestrictions(DetectorRestrictions re } else { - this.logger.LogWarning($"The detector '{id}' has been phased out, we will run the '{this.newDetectorId}' detector which replaced its functionality."); + this.logger.LogWarning("The detector '{OldId}' has been phased out, we will run the '{NewId}' detector which replaced its functionality.", id, this.newDetectorId); } } } diff --git a/src/Microsoft.ComponentDetection.Orchestrator/Services/GraphTranslation/DefaultGraphTranslationService.cs b/src/Microsoft.ComponentDetection.Orchestrator/Services/GraphTranslation/DefaultGraphTranslationService.cs index cf24d199a..168b8a23e 100644 --- a/src/Microsoft.ComponentDetection.Orchestrator/Services/GraphTranslation/DefaultGraphTranslationService.cs +++ b/src/Microsoft.ComponentDetection.Orchestrator/Services/GraphTranslation/DefaultGraphTranslationService.cs @@ -11,10 +11,13 @@ namespace Microsoft.ComponentDetection.Orchestrator.Services.GraphTranslation; using Microsoft.ComponentDetection.Contracts.BcdeModels; using Microsoft.ComponentDetection.Contracts.TypedComponent; using Microsoft.ComponentDetection.Orchestrator.ArgumentSets; +using Microsoft.Extensions.Logging; -public class DefaultGraphTranslationService : ServiceBase, IGraphTranslationService +public class DefaultGraphTranslationService : IGraphTranslationService { - public DefaultGraphTranslationService(ILogger logger) => this.Logger = logger; + private readonly ILogger logger; + + public DefaultGraphTranslationService(ILogger logger) => this.logger = logger; public ScanResult GenerateScanResultFromProcessingResult(DetectorProcessingResult detectorProcessingResult, IDetectionArguments detectionArguments) { @@ -86,7 +89,7 @@ private IEnumerable GatherSetOfDetectedComponentsUnmerged(IEn var locations = dependencyGraph.GetAdditionalRelatedFiles(); locations.Add(location); - var relativePaths = this.MakeFilePathsRelative(this.Logger, rootDirectory, locations); + var relativePaths = this.MakeFilePathsRelative(this.logger, rootDirectory, locations); foreach (var additionalRelatedFile in relativePaths ?? Enumerable.Empty()) { @@ -207,9 +210,9 @@ private HashSet MakeFilePathsRelative(ILogger logger, DirectoryInfo root relativePathSet.Add(relativePath); } - catch (UriFormatException) + catch (UriFormatException e) { - logger.LogVerbose($"The path: {path} could not be resolved relative to the root {rootUri}"); + logger.LogDebug(e, "The path: {Path} could not be resolved relative to the root {RootUri}", path, rootUri); } } diff --git a/src/Microsoft.ComponentDetection.Orchestrator/Services/ServiceBase.cs b/src/Microsoft.ComponentDetection.Orchestrator/Services/ServiceBase.cs deleted file mode 100644 index 8174964de..000000000 --- a/src/Microsoft.ComponentDetection.Orchestrator/Services/ServiceBase.cs +++ /dev/null @@ -1,7 +0,0 @@ -namespace Microsoft.ComponentDetection.Orchestrator.Services; -using Microsoft.ComponentDetection.Contracts; - -public abstract class ServiceBase -{ - protected ILogger Logger { get; set; } -} diff --git a/src/Microsoft.ComponentDetection/Microsoft.ComponentDetection.csproj b/src/Microsoft.ComponentDetection/Microsoft.ComponentDetection.csproj index 642f40c5b..fe4db3577 100644 --- a/src/Microsoft.ComponentDetection/Microsoft.ComponentDetection.csproj +++ b/src/Microsoft.ComponentDetection/Microsoft.ComponentDetection.csproj @@ -8,6 +8,10 @@ + + + + diff --git a/src/Microsoft.ComponentDetection/Program.cs b/src/Microsoft.ComponentDetection/Program.cs index 752b88b93..d1a1fa01e 100644 --- a/src/Microsoft.ComponentDetection/Program.cs +++ b/src/Microsoft.ComponentDetection/Program.cs @@ -6,6 +6,7 @@ using Microsoft.ComponentDetection.Orchestrator; using Microsoft.ComponentDetection.Orchestrator.Extensions; using Microsoft.Extensions.DependencyInjection; +using Serilog; try { @@ -18,8 +19,13 @@ } } + Log.Logger = new LoggerConfiguration() + .WriteTo.Console(outputTemplate: "[BOOTSTRAP] [{Timestamp:HH:mm:ss} {Level:u3}] {Message:lj}{NewLine}{Exception}") + .CreateBootstrapLogger(); + var serviceProvider = new ServiceCollection() .AddComponentDetection() + .ConfigureLoggingProviders() .BuildServiceProvider(); var orchestrator = serviceProvider.GetRequiredService(); var result = await orchestrator.LoadAsync(args); @@ -35,6 +41,8 @@ // Manually dispose to flush logs as we force exit await serviceProvider.DisposeAsync(); + await Log.CloseAndFlushAsync(); + // force an exit, not letting any lingering threads not responding. Environment.Exit(exitCode); } diff --git a/test/Microsoft.ComponentDetection.Common.Tests/ComponentStreamEnumerableTests.cs b/test/Microsoft.ComponentDetection.Common.Tests/ComponentStreamEnumerableTests.cs index 299d84b12..4bf367b07 100644 --- a/test/Microsoft.ComponentDetection.Common.Tests/ComponentStreamEnumerableTests.cs +++ b/test/Microsoft.ComponentDetection.Common.Tests/ComponentStreamEnumerableTests.cs @@ -1,8 +1,10 @@ namespace Microsoft.ComponentDetection.Common.Tests; + +using System; using System.IO; using System.Linq; using FluentAssertions; -using Microsoft.ComponentDetection.Contracts; +using Microsoft.Extensions.Logging; using Microsoft.VisualStudio.TestTools.UnitTesting; using Moq; @@ -60,7 +62,12 @@ public void GetEnumerator_LogsAndBreaksEnumerationWhenFileIsMissing() var tempFileTwo = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName()); var tempFileThree = Path.GetTempFileName(); File.Delete(tempFileTwo); - this.loggerMock.Setup(x => x.LogWarning(Match.Create(message => message.Contains("not exist")))); + this.loggerMock.Setup(x => x.Log( + LogLevel.Warning, + It.IsAny(), + It.Is((v, t) => v.ToString().Contains(tempFileTwo)), + It.IsAny(), + It.IsAny>())); var enumerable = new ComponentStreamEnumerable( new[] { diff --git a/test/Microsoft.ComponentDetection.Common.Tests/DockerServiceTests.cs b/test/Microsoft.ComponentDetection.Common.Tests/DockerServiceTests.cs index a3a744800..7d3701901 100644 --- a/test/Microsoft.ComponentDetection.Common.Tests/DockerServiceTests.cs +++ b/test/Microsoft.ComponentDetection.Common.Tests/DockerServiceTests.cs @@ -1,10 +1,11 @@ namespace Microsoft.ComponentDetection.Common.Tests; + using System; using System.Collections.Generic; using System.Threading.Tasks; using FluentAssertions; -using Microsoft.ComponentDetection.Contracts; using Microsoft.ComponentDetection.TestsUtilities; +using Microsoft.Extensions.Logging; using Microsoft.VisualStudio.TestTools.UnitTesting; using Moq; @@ -17,7 +18,7 @@ public class DockerServiceTests private const string TestImageWithBaseDetails = "governancecontainerregistry.azurecr.io/testcontainers/dockertags_test:testtag"; - private readonly Mock loggerMock = new(); + private readonly Mock> loggerMock = new(); private readonly DockerService dockerService; public DockerServiceTests() => this.dockerService = new DockerService(this.loggerMock.Object); diff --git a/test/Microsoft.ComponentDetection.Common.Tests/FileEnumerationTests.cs b/test/Microsoft.ComponentDetection.Common.Tests/FileEnumerationTests.cs index 3c1fd52d1..babec5554 100644 --- a/test/Microsoft.ComponentDetection.Common.Tests/FileEnumerationTests.cs +++ b/test/Microsoft.ComponentDetection.Common.Tests/FileEnumerationTests.cs @@ -1,9 +1,10 @@ namespace Microsoft.ComponentDetection.Common.Tests; + using System; using System.Collections.Generic; using System.IO; using System.Runtime.InteropServices; -using Microsoft.ComponentDetection.Contracts; +using Microsoft.Extensions.Logging; using Microsoft.VisualStudio.TestTools.UnitTesting; using Moq; @@ -21,7 +22,7 @@ public void CanListAllFiles() Assert.Inconclusive("Test directory environment variable isn't set. Not testing"); } - var loggerMock = new Mock(); + var loggerMock = new Mock>(); var pathUtility = new PathUtilityService(loggerMock.Object); var sfe = new SafeFileEnumerable(new DirectoryInfo(Path.Combine(testDirectory, "root")), new[] { "*" }, loggerMock.Object, pathUtility, (name, directoryName) => false, true); diff --git a/test/Microsoft.ComponentDetection.Common.Tests/LoggerTests.cs b/test/Microsoft.ComponentDetection.Common.Tests/LoggerTests.cs deleted file mode 100644 index e098309e9..000000000 --- a/test/Microsoft.ComponentDetection.Common.Tests/LoggerTests.cs +++ /dev/null @@ -1,272 +0,0 @@ -namespace Microsoft.ComponentDetection.Common.Tests; -using System; -using Microsoft.ComponentDetection.Contracts; -using Microsoft.VisualStudio.TestTools.UnitTesting; -using Moq; - -[TestClass] -[TestCategory("Governance/All")] -[TestCategory("Governance/ComponentDetection")] -public class LoggerTests -{ - private readonly Mock fileWritingServiceMock; - private readonly Mock consoleWritingServiceMock; - - public LoggerTests() - { - this.consoleWritingServiceMock = new Mock(); - this.fileWritingServiceMock = new Mock(); - } - - [TestCleanup] - public void TestCleanup() - { - this.consoleWritingServiceMock.VerifyAll(); - this.fileWritingServiceMock.VerifyAll(); - } - - [TestMethod] - public void LogCreateLoggingGroup_HandlesFailedInit() - { - var logger = new Logger(this.consoleWritingServiceMock.Object, null); - - // This should throw an exception while setting up the file writing service, but handle it - logger.Init(VerbosityMode.Normal); - - this.consoleWritingServiceMock.Invocations.Clear(); - this.consoleWritingServiceMock.Setup(x => x.Write(Environment.NewLine)); - - // This should not fail, despite not initializing the file writing service - logger.LogCreateLoggingGroup(); - - // As a result of handling the file writing service failure, the verbosity should now be Verbose - var verboseMessage = "verboseMessage"; - var expectedMessage = $"[VERBOSE] {verboseMessage} {Environment.NewLine}"; - this.consoleWritingServiceMock.Setup(x => x.Write(expectedMessage)); - - logger.LogVerbose(verboseMessage); - } - - [TestMethod] - public void LogCreateLoggingGroup_WritesOnNormal() - { - var logger = this.CreateLogger(VerbosityMode.Normal); - this.consoleWritingServiceMock.Setup(x => x.Write(Environment.NewLine)); - this.fileWritingServiceMock.Setup(x => x.AppendToFile(Logger.LogRelativePath, Environment.NewLine)); - logger.LogCreateLoggingGroup(); - } - - [TestMethod] - public void LogCreateLoggingGroup_SkipsConsoleOnQuiet() - { - var logger = this.CreateLogger(VerbosityMode.Quiet); - this.fileWritingServiceMock.Setup(x => x.AppendToFile(Logger.LogRelativePath, Environment.NewLine)); - logger.LogCreateLoggingGroup(); - } - - [TestMethod] - public void LogWarning_WritesOnNormal() - { - var logger = this.CreateLogger(VerbosityMode.Normal); - var warningMessage = "warningMessage"; - var expectedMessage = $"[WARN] {warningMessage} {Environment.NewLine}"; - this.consoleWritingServiceMock.Setup(x => x.Write(expectedMessage)); - this.fileWritingServiceMock.Setup(x => x.AppendToFile(Logger.LogRelativePath, expectedMessage)); - logger.LogWarning(warningMessage); - } - - [TestMethod] - public void LogWarning_SkipsConsoleOnQuiet() - { - var logger = this.CreateLogger(VerbosityMode.Quiet); - var warningMessage = "warningMessage"; - var expectedMessage = $"[WARN] {warningMessage} {Environment.NewLine}"; - this.fileWritingServiceMock.Setup(x => x.AppendToFile(Logger.LogRelativePath, expectedMessage)); - logger.LogWarning(warningMessage); - } - - [TestMethod] - public void LogInfo_WritesOnNormal() - { - var logger = this.CreateLogger(VerbosityMode.Normal); - var infoMessage = "informationalMessage"; - var expectedMessage = $"[INFO] {infoMessage} {Environment.NewLine}"; - this.consoleWritingServiceMock.Setup(x => x.Write(expectedMessage)); - this.fileWritingServiceMock.Setup(x => x.AppendToFile(Logger.LogRelativePath, expectedMessage)); - logger.LogInfo(infoMessage); - } - - [TestMethod] - public void LogInfo_SkipsConsoleOnQuiet() - { - var logger = this.CreateLogger(VerbosityMode.Quiet); - var infoMessage = "informationalMessage"; - var expectedMessage = $"[INFO] {infoMessage} {Environment.NewLine}"; - this.fileWritingServiceMock.Setup(x => x.AppendToFile(Logger.LogRelativePath, expectedMessage)); - logger.LogInfo(infoMessage); - } - - [TestMethod] - public void LogVerbose_WritesOnVerbose() - { - var logger = this.CreateLogger(VerbosityMode.Verbose); - var verboseMessage = "verboseMessage"; - var expectedMessage = $"[VERBOSE] {verboseMessage} {Environment.NewLine}"; - this.consoleWritingServiceMock.Setup(x => x.Write(expectedMessage)); - this.fileWritingServiceMock.Setup(x => x.AppendToFile(Logger.LogRelativePath, expectedMessage)); - logger.LogVerbose(verboseMessage); - } - - [TestMethod] - public void LogVerbose_SkipsConsoleOnNormal() - { - var logger = this.CreateLogger(VerbosityMode.Normal); - var verboseMessage = "verboseMessage"; - var expectedMessage = $"[VERBOSE] {verboseMessage} {Environment.NewLine}"; - this.fileWritingServiceMock.Setup(x => x.AppendToFile(Logger.LogRelativePath, expectedMessage)); - logger.LogVerbose(verboseMessage); - } - - [TestMethod] - public void LogError_WritesOnQuiet() - { - var logger = this.CreateLogger(VerbosityMode.Quiet); - var errorMessage = "errorMessage"; - var expectedMessage = $"[ERROR] {errorMessage} {Environment.NewLine}"; - this.consoleWritingServiceMock.Setup(x => x.Write(expectedMessage)); - this.fileWritingServiceMock.Setup(x => x.AppendToFile(Logger.LogRelativePath, expectedMessage)); - logger.LogError(errorMessage); - } - - [TestMethod] - public void LogFailedReadingFile_WritesOnVerbose() - { - var logger = this.CreateLogger(VerbosityMode.Verbose); - var filePath = "some/bad/file/path"; - var error = new UnauthorizedAccessException("Some unauthorized access error"); - - var consoleSequence = new MockSequence(); - this.consoleWritingServiceMock.InSequence(consoleSequence).Setup(x => x.Write(Environment.NewLine)); - this.consoleWritingServiceMock.InSequence(consoleSequence).Setup(x => x.Write( - Match.Create(message => message.StartsWith("[VERBOSE]") && message.Contains(filePath)))); - this.consoleWritingServiceMock.InSequence(consoleSequence).Setup(x => x.Write( - Match.Create(message => message.StartsWith("[INFO]") && message.Contains(error.Message)))); - - var fileSequence = new MockSequence(); - this.fileWritingServiceMock.InSequence(fileSequence).Setup(x => x.AppendToFile( - Logger.LogRelativePath, - Match.Create(message => message.StartsWith("[VERBOSE]") && message.Contains(filePath)))); - this.fileWritingServiceMock.InSequence(fileSequence).Setup(x => x.AppendToFile( - Logger.LogRelativePath, - Match.Create(message => message.StartsWith("[INFO]") && message.Contains(error.Message)))); - - logger.LogFailedReadingFile(filePath, error); - } - - [TestMethod] - public void LogFailedReadingFile_SkipsConsoleOnQuiet() - { - var logger = this.CreateLogger(VerbosityMode.Quiet); - var filePath = "some/bad/file/path"; - var error = new UnauthorizedAccessException("Some unauthorized access error"); - - var fileSequence = new MockSequence(); - this.fileWritingServiceMock.InSequence(fileSequence).Setup(x => x.AppendToFile( - Logger.LogRelativePath, - Match.Create(message => message.StartsWith("[VERBOSE]") && message.Contains(filePath)))); - this.fileWritingServiceMock.InSequence(fileSequence).Setup(x => x.AppendToFile( - Logger.LogRelativePath, - Match.Create(message => message.StartsWith("[INFO]") && message.Contains(error.Message)))); - - logger.LogFailedReadingFile(filePath, error); - } - - [TestMethod] - public void LogException_WritesOnQuietIfError() - { - var logger = this.CreateLogger(VerbosityMode.Quiet); - var error = new UnauthorizedAccessException("Some unauthorized access error"); - - this.consoleWritingServiceMock.Setup(x => x.Write( - Match.Create(message => message.StartsWith("[ERROR]") && message.Contains(error.Message)))); - - this.fileWritingServiceMock.Setup(x => x.AppendToFile( - Logger.LogRelativePath, - Match.Create(message => message.StartsWith("[ERROR]") && message.Contains(error.ToString())))); - - logger.LogException(error, true); - } - - [TestMethod] - public void LogException_DoesNotLogFullExceptionByDefault() - { - var logger = this.CreateLogger(VerbosityMode.Quiet); - var error = new UnauthorizedAccessException("Some unauthorized access error"); - - this.consoleWritingServiceMock.Setup(x => x.Write( - Match.Create(message => message.StartsWith("[ERROR]") && message.Contains(error.Message) && !message.Contains(error.ToString())))); - - this.fileWritingServiceMock.Setup(x => x.AppendToFile( - Logger.LogRelativePath, - Match.Create(message => message.StartsWith("[ERROR]") && message.Contains(error.ToString())))); - - logger.LogException(error, true); - } - - [TestMethod] - public void LogException_LogsFullExceptionOnRequest() - { - var logger = this.CreateLogger(VerbosityMode.Quiet); - var error = new UnauthorizedAccessException("Some unauthorized access error"); - - this.consoleWritingServiceMock.Setup(x => x.Write( - Match.Create(message => message.StartsWith("[ERROR]") && message.Contains(error.ToString())))); - - this.fileWritingServiceMock.Setup(x => x.AppendToFile( - Logger.LogRelativePath, - Match.Create(message => message.StartsWith("[ERROR]") && message.Contains(error.ToString())))); - - logger.LogException(error, true, printException: true); - } - - [TestMethod] - public void LogException_SkipsConsoleIfNotErrorAndNormalLogging() - { - var logger = this.CreateLogger(VerbosityMode.Normal); - var error = new UnauthorizedAccessException("Some unauthorized access error"); - - this.fileWritingServiceMock.Setup(x => x.AppendToFile( - Logger.LogRelativePath, - Match.Create(message => message.StartsWith("[INFO]") && message.Contains(error.ToString())))); - - logger.LogException(error, false); - } - - [TestMethod] - public void LogException_WritesEverythingIfNotErrorAndVerboseLogging() - { - var logger = this.CreateLogger(VerbosityMode.Verbose); - var error = new UnauthorizedAccessException("Some unauthorized access error"); - - this.consoleWritingServiceMock.Setup(x => x.Write( - Match.Create(message => message.StartsWith("[INFO]") && message.Contains(error.Message)))); - - this.fileWritingServiceMock.Setup(x => x.AppendToFile( - Logger.LogRelativePath, - Match.Create(message => message.StartsWith("[INFO]") && message.Contains(error.Message)))); - - logger.LogException(error, false); - } - - private Logger CreateLogger(VerbosityMode verbosityMode) - { - var serviceUnderTest = new Logger(this.consoleWritingServiceMock.Object, this.fileWritingServiceMock.Object); - - serviceUnderTest.Init(verbosityMode); - - // We're not explicitly testing init behavior here, so we reset mock expecations. Another test should verify these. - this.consoleWritingServiceMock.Invocations.Clear(); - this.fileWritingServiceMock.Invocations.Clear(); - return serviceUnderTest; - } -} diff --git a/test/Microsoft.ComponentDetection.Common.Tests/Microsoft.ComponentDetection.Common.Tests.csproj b/test/Microsoft.ComponentDetection.Common.Tests/Microsoft.ComponentDetection.Common.Tests.csproj index 2ea0fdef1..b63956f6f 100644 --- a/test/Microsoft.ComponentDetection.Common.Tests/Microsoft.ComponentDetection.Common.Tests.csproj +++ b/test/Microsoft.ComponentDetection.Common.Tests/Microsoft.ComponentDetection.Common.Tests.csproj @@ -1,11 +1,12 @@  - + - + + diff --git a/test/Microsoft.ComponentDetection.Common.Tests/SafeFileEnumerableTests.cs b/test/Microsoft.ComponentDetection.Common.Tests/SafeFileEnumerableTests.cs index 7a65c1879..41b0959f1 100644 --- a/test/Microsoft.ComponentDetection.Common.Tests/SafeFileEnumerableTests.cs +++ b/test/Microsoft.ComponentDetection.Common.Tests/SafeFileEnumerableTests.cs @@ -1,9 +1,11 @@ namespace Microsoft.ComponentDetection.Common.Tests; + using System; using System.Collections.Generic; using System.IO; using FluentAssertions; using Microsoft.ComponentDetection.Contracts; +using Microsoft.Extensions.Logging; using Microsoft.VisualStudio.TestTools.UnitTesting; using Moq; diff --git a/test/Microsoft.ComponentDetection.Contracts.Tests/Microsoft.ComponentDetection.Contracts.Tests.csproj b/test/Microsoft.ComponentDetection.Contracts.Tests/Microsoft.ComponentDetection.Contracts.Tests.csproj index 226491082..ddcc3a6ad 100644 --- a/test/Microsoft.ComponentDetection.Contracts.Tests/Microsoft.ComponentDetection.Contracts.Tests.csproj +++ b/test/Microsoft.ComponentDetection.Contracts.Tests/Microsoft.ComponentDetection.Contracts.Tests.csproj @@ -1,14 +1,15 @@ - + - - + + + - + diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/LinuxContainerDetectorTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/LinuxContainerDetectorTests.cs index 1f996d3af..541266944 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/LinuxContainerDetectorTests.cs +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/LinuxContainerDetectorTests.cs @@ -1,4 +1,5 @@ namespace Microsoft.ComponentDetection.Detectors.Tests; + using System; using System.Collections.Generic; using System.IO; @@ -11,6 +12,7 @@ namespace Microsoft.ComponentDetection.Detectors.Tests; using Microsoft.ComponentDetection.Contracts.BcdeModels; using Microsoft.ComponentDetection.Contracts.TypedComponent; using Microsoft.ComponentDetection.Detectors.Linux; +using Microsoft.Extensions.Logging; using Microsoft.VisualStudio.TestTools.UnitTesting; using Moq; @@ -34,6 +36,7 @@ public class LinuxContainerDetectorTests private readonly Mock mockDockerService; private readonly Mock mockLogger; + private readonly Mock> mockLinuxContainerDetectorLogger; private readonly Mock mockSyftLinuxScanner; public LinuxContainerDetectorTests() @@ -47,6 +50,7 @@ public LinuxContainerDetectorTests() .ReturnsAsync(new ContainerDetails { Id = 1, ImageId = NodeLatestDigest, Layers = Enumerable.Empty() }); this.mockLogger = new Mock(); + this.mockLinuxContainerDetectorLogger = new Mock>(); this.mockSyftLinuxScanner = new Mock(); this.mockSyftLinuxScanner.Setup(scanner => scanner.ScanLinuxAsync(It.IsAny(), It.IsAny>(), It.IsAny(), It.IsAny())) @@ -63,7 +67,7 @@ public async Task TestLinuxContainerDetectorAsync() var linuxContainerDetector = new LinuxContainerDetector( this.mockSyftLinuxScanner.Object, this.mockDockerService.Object, - this.mockLogger.Object); + this.mockLinuxContainerDetectorLogger.Object); var scanResult = await linuxContainerDetector.ExecuteDetectorAsync(scanRequest); @@ -91,7 +95,7 @@ public async Task TestLinuxContainerDetector_CantRunLinuxContainersAsync() var linuxContainerDetector = new LinuxContainerDetector( this.mockSyftLinuxScanner.Object, this.mockDockerService.Object, - this.mockLogger.Object); + this.mockLinuxContainerDetectorLogger.Object); var scanResult = await linuxContainerDetector.ExecuteDetectorAsync(scanRequest); @@ -100,7 +104,12 @@ public async Task TestLinuxContainerDetector_CantRunLinuxContainersAsync() scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); detectedComponents.Should().HaveCount(0); scanResult.ContainerDetails.Should().HaveCount(0); - this.mockLogger.Verify(logger => logger.LogInfo(It.IsAny())); + this.mockLinuxContainerDetectorLogger.Verify(logger => logger.Log( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + (Func)It.IsAny())); } [TestMethod] @@ -113,7 +122,7 @@ public async Task TestLinuxContainerDetector_TestNullAsync() var linuxContainerDetector = new LinuxContainerDetector( this.mockSyftLinuxScanner.Object, this.mockDockerService.Object, - this.mockLogger.Object); + this.mockLinuxContainerDetectorLogger.Object); var scanResult = await linuxContainerDetector.ExecuteDetectorAsync(scanRequest); @@ -122,7 +131,12 @@ public async Task TestLinuxContainerDetector_TestNullAsync() scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); detectedComponents.Should().HaveCount(0); scanResult.ContainerDetails.Should().HaveCount(0); - this.mockLogger.Verify(logger => logger.LogInfo(It.IsAny())); + this.mockLinuxContainerDetectorLogger.Verify(logger => logger.Log( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + (Func)It.IsAny())); } [TestMethod] @@ -135,7 +149,7 @@ public async Task TestLinuxContainerDetector_VerifyLowerCaseAsync() var linuxContainerDetector = new LinuxContainerDetector( this.mockSyftLinuxScanner.Object, this.mockDockerService.Object, - this.mockLogger.Object); + this.mockLinuxContainerDetectorLogger.Object); var scanResult = await linuxContainerDetector.ExecuteDetectorAsync(scanRequest); @@ -158,7 +172,7 @@ public async Task TestLinuxContainerDetector_SameImagePassedMultipleTimesAsync() var linuxContainerDetector = new LinuxContainerDetector( this.mockSyftLinuxScanner.Object, this.mockDockerService.Object, - this.mockLogger.Object); + this.mockLinuxContainerDetectorLogger.Object); var scanResult = await linuxContainerDetector.ExecuteDetectorAsync(scanRequest); @@ -181,7 +195,7 @@ public async Task TestLinuxContainerDetector_TimeoutParameterSpecifiedAsync() var linuxContainerDetector = new LinuxContainerDetector( this.mockSyftLinuxScanner.Object, this.mockDockerService.Object, - this.mockLogger.Object); + this.mockLinuxContainerDetectorLogger.Object); Func action = async () => await linuxContainerDetector.ExecuteDetectorAsync(scanRequest); await action.Should().NotThrowAsync(); diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/LinuxScannerTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/LinuxScannerTests.cs index 5e989d465..1108f80bf 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/LinuxScannerTests.cs +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/LinuxScannerTests.cs @@ -1,4 +1,5 @@ namespace Microsoft.ComponentDetection.Detectors.Tests; + using System.Collections.Generic; using System.Linq; using System.Threading; @@ -7,6 +8,7 @@ namespace Microsoft.ComponentDetection.Detectors.Tests; using Microsoft.ComponentDetection.Contracts; using Microsoft.ComponentDetection.Contracts.BcdeModels; using Microsoft.ComponentDetection.Detectors.Linux; +using Microsoft.Extensions.Logging; using Microsoft.VisualStudio.TestTools.UnitTesting; using Moq; @@ -37,7 +39,7 @@ public class LinuxScannerTests private readonly LinuxScanner linuxScanner; private readonly Mock mockDockerService; - private readonly Mock mockLogger; + private readonly Mock> mockLogger; public LinuxScannerTests() { @@ -48,7 +50,7 @@ public LinuxScannerTests() this.mockDockerService.Setup(service => service.CreateAndRunContainerAsync(It.IsAny(), It.IsAny>(), It.IsAny())) .ReturnsAsync((SyftOutput, string.Empty)); - this.mockLogger = new Mock(); + this.mockLogger = new Mock>(); this.linuxScanner = new LinuxScanner(this.mockDockerService.Object, this.mockLogger.Object); } diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/MavenCommandServiceTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/MavenCommandServiceTests.cs index 481f22668..aad02717b 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/MavenCommandServiceTests.cs +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/MavenCommandServiceTests.cs @@ -1,4 +1,5 @@ namespace Microsoft.ComponentDetection.Detectors.Tests; + using System; using System.Collections.Generic; using System.IO; @@ -9,6 +10,7 @@ using Microsoft.ComponentDetection.Contracts; using Microsoft.ComponentDetection.Contracts.Internal; using Microsoft.ComponentDetection.Detectors.Maven; +using Microsoft.Extensions.Logging; using Microsoft.VisualStudio.TestTools.UnitTesting; using Moq; @@ -24,7 +26,7 @@ public class MavenCommandServiceTests public MavenCommandServiceTests() { this.commandLineMock = new Mock(); - var loggerMock = new Mock(); + var loggerMock = new Mock>(); this.parserServiceMock = new Mock(); diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/Microsoft.ComponentDetection.Detectors.Tests.csproj b/test/Microsoft.ComponentDetection.Detectors.Tests/Microsoft.ComponentDetection.Detectors.Tests.csproj index 376bc6372..2138b5b46 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/Microsoft.ComponentDetection.Detectors.Tests.csproj +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/Microsoft.ComponentDetection.Detectors.Tests.csproj @@ -5,6 +5,7 @@ + diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/NpmUtilitiesTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/NpmUtilitiesTests.cs index 42f581f11..cc2fc9114 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/NpmUtilitiesTests.cs +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/NpmUtilitiesTests.cs @@ -1,4 +1,5 @@ namespace Microsoft.ComponentDetection.Detectors.Tests; + using System.Linq; using FluentAssertions; using Microsoft.ComponentDetection.Common.DependencyGraph; @@ -6,6 +7,7 @@ using Microsoft.ComponentDetection.Contracts.TypedComponent; using Microsoft.ComponentDetection.Detectors.Npm; using Microsoft.ComponentDetection.Detectors.Tests.Utilities; +using Microsoft.Extensions.Logging; using Microsoft.VisualStudio.TestTools.UnitTesting; using Moq; using Newtonsoft.Json.Linq; @@ -251,7 +253,7 @@ public void AddOrUpdateDetectedComponent_ComponentExistAsDevDependencyNewUpdateI DevelopmentDependency = true, }; - var componentRecorder = new ComponentRecorder(); + var componentRecorder = new ComponentRecorder(new Mock().Object); var singleFileComponentRecorder = componentRecorder.CreateSingleFileComponentRecorder("path"); singleFileComponentRecorder.RegisterUsage(detectedComponent); diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/NuGetComponentDetectorTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/NuGetComponentDetectorTests.cs index 3040a5756..29c24ba69 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/NuGetComponentDetectorTests.cs +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/NuGetComponentDetectorTests.cs @@ -13,6 +13,7 @@ using Microsoft.ComponentDetection.Contracts.TypedComponent; using Microsoft.ComponentDetection.Detectors.NuGet; using Microsoft.ComponentDetection.TestsUtilities; +using Microsoft.Extensions.Logging; using Microsoft.VisualStudio.TestTools.UnitTesting; using Moq; @@ -24,6 +25,14 @@ public class NuGetComponentDetectorTests : BaseDetectorTest DetectorSearchPattern = new List { "*.nupkg", "*.nuspec", "nuget.config", "paket.lock" }; + private readonly Mock> mockLogger; + + public NuGetComponentDetectorTests() + { + this.mockLogger = new Mock>(); + this.DetectorTestUtility.AddServiceMock(this.mockLogger); + } + [TestMethod] public async Task TestNuGetDetectorWithNoFiles_ReturnsSuccessfullyAsync() { @@ -157,16 +166,19 @@ public async Task TestNugetDetector_HandlesMalformedComponentsInComponentListAsy var malformedNupkg = await NugetTestUtilities.ZipNupkgComponentAsync("malformed.nupkg", NugetTestUtilities.GetRandomMalformedNuPkgComponent()); var nuspec = NugetTestUtilities.GetRandomValidNuSpecComponent(); - var mockLogger = new Mock(); - var (scanResult, componentRecorder) = await this.DetectorTestUtility .WithFile("test.nuspec", nuspec) .WithFile("test.nupkg", validNupkg) .WithFile("malformed.nupkg", malformedNupkg) - .AddServiceMock(mockLogger) + .AddServiceMock(this.mockLogger) .ExecuteDetectorAsync(); - mockLogger.Verify(x => x.LogFailedReadingFile(Path.Join(Path.GetTempPath(), "malformed.nupkg"), It.IsAny())); + this.mockLogger.Verify(x => x.Log( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + (Func)It.IsAny())); Assert.AreEqual(ProcessingResultCode.Success, scanResult.ResultCode); Assert.AreEqual(2, componentRecorder.GetDetectedComponents().Count()); @@ -239,7 +251,7 @@ public async Task TestNugetDetector_AdditionalDirectoriesAsync() var detector = new NuGetComponentDetector( componentStreamEnumerableFactoryMock.Object, directoryWalkerMock.Object, - mockLogger.Object); + new Mock>().Object); var scanResult = await detector.ExecuteDetectorAsync(new ScanRequest(new DirectoryInfo(sourceDirectoryPath), (name, directoryName) => false, null, new Dictionary(), null, componentRecorder)); diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/PipComponentDetectorTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/PipComponentDetectorTests.cs index 8b591fe3b..1ddc83cb6 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/PipComponentDetectorTests.cs +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/PipComponentDetectorTests.cs @@ -11,6 +11,7 @@ namespace Microsoft.ComponentDetection.Detectors.Tests; using Microsoft.ComponentDetection.Detectors.Pip; using Microsoft.ComponentDetection.Detectors.Tests.Utilities; using Microsoft.ComponentDetection.TestsUtilities; +using Microsoft.Extensions.Logging; using Microsoft.VisualStudio.TestTools.UnitTesting; using Moq; using Newtonsoft.Json; @@ -20,6 +21,7 @@ public class PipComponentDetectorTests : BaseDetectorTest { private readonly Mock pythonCommandService; private readonly Mock pythonResolver; + private readonly Mock> mockLogger; public PipComponentDetectorTests() { @@ -28,14 +30,22 @@ public PipComponentDetectorTests() this.pythonResolver = new Mock(); this.DetectorTestUtility.AddServiceMock(this.pythonResolver); + + this.mockLogger = new Mock>(); + this.DetectorTestUtility.AddServiceMock(this.mockLogger); } [TestMethod] public async Task TestPipDetector_PythonNotInstalledAsync() { - var mockLogger = new Mock(); - mockLogger.Setup(x => x.LogInfo(It.Is(l => l.Contains("No python found")))); - this.DetectorTestUtility.AddServiceMock(mockLogger); + this.mockLogger.Setup(x => x.Log( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + (Func)It.IsAny())); + + this.DetectorTestUtility.AddServiceMock(this.mockLogger); this.pythonCommandService.Setup(x => x.PythonExistsAsync(It.IsAny())).ReturnsAsync(false); @@ -44,7 +54,7 @@ public async Task TestPipDetector_PythonNotInstalledAsync() .ExecuteDetectorAsync(); Assert.AreEqual(ProcessingResultCode.Success, result.ResultCode); - mockLogger.VerifyAll(); + this.mockLogger.VerifyAll(); } [TestMethod] diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/PipResolverTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/PipResolverTests.cs index 756da8f2b..78882cfbe 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/PipResolverTests.cs +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/PipResolverTests.cs @@ -1,4 +1,5 @@ namespace Microsoft.ComponentDetection.Detectors.Tests; + using System; using System.Collections.Generic; using System.Linq; @@ -6,6 +7,7 @@ namespace Microsoft.ComponentDetection.Detectors.Tests; using Microsoft.ComponentDetection.Contracts; using Microsoft.ComponentDetection.Contracts.TypedComponent; using Microsoft.ComponentDetection.Detectors.Pip; +using Microsoft.Extensions.Logging; using Microsoft.VisualStudio.TestTools.UnitTesting; using Moq; @@ -14,14 +16,14 @@ namespace Microsoft.ComponentDetection.Detectors.Tests; [TestCategory("Governance/ComponentDetection")] public class PipResolverTests { - private Mock loggerMock; + private Mock> loggerMock; private Mock pyPiClient; private Mock recorderMock; [TestInitialize] public void TestInitialize() { - this.loggerMock = new Mock(); + this.loggerMock = new Mock>(); this.pyPiClient = new Mock(); this.recorderMock = new Mock(); } diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/PnpmDetectorTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/PnpmDetectorTests.cs index 7e398b7c3..04e1158b3 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/PnpmDetectorTests.cs +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/PnpmDetectorTests.cs @@ -11,7 +11,9 @@ using Microsoft.ComponentDetection.Detectors.Pnpm; using Microsoft.ComponentDetection.Detectors.Tests.Utilities; using Microsoft.ComponentDetection.TestsUtilities; +using Microsoft.Extensions.Logging; using Microsoft.VisualStudio.TestTools.UnitTesting; +using Moq; [TestClass] [TestCategory("Governance/All")] @@ -29,6 +31,7 @@ public PnpmDetectorTests() new Dictionary(), null, componentRecorder)); + this.DetectorTestUtility.AddServiceMock(new Mock>()); } [TestMethod] diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/PyPiClientTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/PyPiClientTests.cs index f473077fa..8c2760e08 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/PyPiClientTests.cs +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/PyPiClientTests.cs @@ -1,4 +1,5 @@ namespace Microsoft.ComponentDetection.Detectors.Tests; + using System; using System.Collections.Generic; using System.IO; @@ -10,6 +11,7 @@ using Microsoft.ComponentDetection.Common; using Microsoft.ComponentDetection.Contracts; using Microsoft.ComponentDetection.Detectors.Pip; +using Microsoft.Extensions.Logging; using Microsoft.VisualStudio.TestTools.UnitTesting; using Moq; using Moq.Protected; @@ -22,7 +24,7 @@ public class PyPiClientTests public PyPiClientTests() => this.pypiClient = new PyPiClient( new EnvironmentVariableService(), - new Mock().Object); + new Mock>().Object); [TestMethod] public async Task GetReleases_InvalidSpecVersion_NotThrowAsync() @@ -158,7 +160,7 @@ public async Task GetReleases_MaxEntriesVariable_CreatesNewCacheAsync() var mockHandler = this.MockHttpMessageHandler(JsonConvert.SerializeObject(pythonProject)); PyPiClient.HttpClient = new HttpClient(mockHandler.Object); - var mockLogger = new Mock(); + var mockLogger = new Mock>(); var mockEvs = new Mock(); mockEvs.Setup(x => x.GetEnvironmentVariable(It.Is(s => s.Equals("PyPiMaxCacheEntries")))).Returns("32"); @@ -173,7 +175,14 @@ public async Task GetReleases_MaxEntriesVariable_CreatesNewCacheAsync() // Verify the cache setup call was performed only once mockEvs.Verify(x => x.GetEnvironmentVariable(It.IsAny()), Times.Once()); - mockLogger.Verify(x => x.LogInfo(It.Is(s => s.Equals("Setting IPyPiClient max cache entries to 32"))), Times.Once()); + mockLogger.Verify( + x => x.Log( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + (Func)It.IsAny()), + Times.Exactly(3)); } private Mock MockHttpMessageHandler(string content) diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/YarnLockDetectorTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/YarnLockDetectorTests.cs index ed80ca71d..70553ec77 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/YarnLockDetectorTests.cs +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/YarnLockDetectorTests.cs @@ -1,4 +1,5 @@ namespace Microsoft.ComponentDetection.Detectors.Tests; + using System; using System.Collections.Generic; using System.IO; @@ -13,6 +14,7 @@ namespace Microsoft.ComponentDetection.Detectors.Tests; using Microsoft.ComponentDetection.Detectors.Yarn; using Microsoft.ComponentDetection.Detectors.Yarn.Parsers; using Microsoft.ComponentDetection.TestsUtilities; +using Microsoft.Extensions.Logging; using Microsoft.VisualStudio.TestTools.UnitTesting; using Moq; using Newtonsoft.Json; @@ -29,11 +31,11 @@ public class YarnLockDetectorTests : BaseDetectorTest public YarnLockDetectorTests() { // TODO: Mock all of this correctly - var loggerMock = new Mock(); + var loggerMock = new Mock>(); this.yarnLockParser = new YarnLockParser(loggerMock.Object); this.yarnLockFileFactory = new YarnLockFileFactory(new[] { this.yarnLockParser }); - var yarnLockFileFactoryMock = new Mock(); + var yarnLockFileFactoryMock = new Mock(); var recorderMock = new Mock(); yarnLockFileFactoryMock.Setup(x => x.ParseYarnLockFileAsync(It.IsAny(), It.IsAny(), It.IsAny())) diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/YarnParserTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/YarnParserTests.cs index 8bdaa515f..e7ff7c6c7 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/YarnParserTests.cs +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/YarnParserTests.cs @@ -1,10 +1,12 @@ namespace Microsoft.ComponentDetection.Detectors.Tests; + using System; using System.Collections.Generic; using System.Linq; using Microsoft.ComponentDetection.Contracts; using Microsoft.ComponentDetection.Detectors.Yarn; using Microsoft.ComponentDetection.Detectors.Yarn.Parsers; +using Microsoft.Extensions.Logging; using Microsoft.VisualStudio.TestTools.UnitTesting; using Moq; @@ -13,12 +15,12 @@ namespace Microsoft.ComponentDetection.Detectors.Tests; [TestCategory("Governance/ComponentDetection")] public class YarnParserTests { - private readonly Mock loggerMock; + private readonly Mock> loggerMock; private readonly Mock recorderMock; public YarnParserTests() { - this.loggerMock = new Mock(); + this.loggerMock = new Mock>(); this.recorderMock = new Mock(); } diff --git a/test/Microsoft.ComponentDetection.Orchestrator.Tests/Microsoft.ComponentDetection.Orchestrator.Tests.csproj b/test/Microsoft.ComponentDetection.Orchestrator.Tests/Microsoft.ComponentDetection.Orchestrator.Tests.csproj index db0632a24..2aa4b970e 100644 --- a/test/Microsoft.ComponentDetection.Orchestrator.Tests/Microsoft.ComponentDetection.Orchestrator.Tests.csproj +++ b/test/Microsoft.ComponentDetection.Orchestrator.Tests/Microsoft.ComponentDetection.Orchestrator.Tests.csproj @@ -1,12 +1,13 @@ - - + + - + + diff --git a/test/Microsoft.ComponentDetection.Orchestrator.Tests/Services/BcdeDevCommandServiceTests.cs b/test/Microsoft.ComponentDetection.Orchestrator.Tests/Services/BcdeDevCommandServiceTests.cs index be75fe05d..bed35f37c 100644 --- a/test/Microsoft.ComponentDetection.Orchestrator.Tests/Services/BcdeDevCommandServiceTests.cs +++ b/test/Microsoft.ComponentDetection.Orchestrator.Tests/Services/BcdeDevCommandServiceTests.cs @@ -7,6 +7,7 @@ using Microsoft.ComponentDetection.Contracts.TypedComponent; using Microsoft.ComponentDetection.Orchestrator.ArgumentSets; using Microsoft.ComponentDetection.Orchestrator.Services; +using Microsoft.Extensions.Logging; using Microsoft.VisualStudio.TestTools.UnitTesting; using Moq; @@ -16,7 +17,7 @@ public class BcdeDevCommandServiceTests { private readonly Mock scanExecutionServiceMock; - private readonly Mock loggerMock; + private readonly Mock> loggerMock; private readonly ScannedComponent[] scannedComponents; @@ -25,7 +26,7 @@ public class BcdeDevCommandServiceTests public BcdeDevCommandServiceTests() { this.scanExecutionServiceMock = new Mock(); - this.loggerMock = new Mock(); + this.loggerMock = new Mock>(); this.serviceUnderTest = new BcdeDevCommandService(this.scanExecutionServiceMock.Object, this.loggerMock.Object); this.scannedComponents = new ScannedComponent[] diff --git a/test/Microsoft.ComponentDetection.Orchestrator.Tests/Services/BcdeScanExecutionServiceTests.cs b/test/Microsoft.ComponentDetection.Orchestrator.Tests/Services/BcdeScanExecutionServiceTests.cs index dd684dae0..c637b5476 100644 --- a/test/Microsoft.ComponentDetection.Orchestrator.Tests/Services/BcdeScanExecutionServiceTests.cs +++ b/test/Microsoft.ComponentDetection.Orchestrator.Tests/Services/BcdeScanExecutionServiceTests.cs @@ -9,11 +9,10 @@ namespace Microsoft.ComponentDetection.Orchestrator.Tests.Services; using Microsoft.ComponentDetection.Contracts; using Microsoft.ComponentDetection.Contracts.BcdeModels; using Microsoft.ComponentDetection.Contracts.TypedComponent; -using Microsoft.ComponentDetection.Detectors.Npm; -using Microsoft.ComponentDetection.Detectors.Pip; using Microsoft.ComponentDetection.Orchestrator.ArgumentSets; using Microsoft.ComponentDetection.Orchestrator.Services; using Microsoft.ComponentDetection.Orchestrator.Services.GraphTranslation; +using Microsoft.Extensions.Logging; using Microsoft.VisualStudio.TestTools.UnitTesting; using Moq; @@ -22,7 +21,7 @@ namespace Microsoft.ComponentDetection.Orchestrator.Tests.Services; [TestCategory("Governance/ComponentDetection")] public class BcdeScanExecutionServiceTests { - private readonly Mock loggerMock; + private readonly Mock> loggerMock; private readonly Mock detectorProcessingServiceMock; private readonly Mock> detectorsMock; private readonly Mock detectorRestrictionServiceMock; @@ -40,7 +39,7 @@ public class BcdeScanExecutionServiceTests public BcdeScanExecutionServiceTests() { - this.loggerMock = new Mock(); + this.loggerMock = new Mock>(); this.detectorProcessingServiceMock = new Mock(); this.detectorsMock = new Mock>(); this.detectorRestrictionServiceMock = new Mock(); @@ -48,7 +47,7 @@ public BcdeScanExecutionServiceTests() this.componentDetector3Mock = new Mock(); this.versionedComponentDetector1Mock = new Mock(); this.sampleContainerDetails = new ContainerDetails { Id = 1 }; - this.graphTranslationService = new DefaultGraphTranslationService(this.loggerMock.Object); + this.graphTranslationService = new DefaultGraphTranslationService(new Mock>().Object); this.detectedComponents = new[] { @@ -87,7 +86,7 @@ public void CleanupTests() [TestMethod] public async Task DetectComponents_HappyPathAsync() { - var componentRecorder = new ComponentRecorder(); + var componentRecorder = new ComponentRecorder(new Mock().Object); var singleFileComponentRecorder = componentRecorder.CreateSingleFileComponentRecorder(Path.Join(this.sourceDirectory.FullName, "/some/file/path")); this.componentDetector2Mock.SetupGet(x => x.Id).Returns("Detector2"); @@ -572,7 +571,7 @@ public async Task VerifyTranslation_RootsAreMergedWhenSameComponentInDifferentFi [TestMethod] public async Task VerifyTranslation_DetectedComponentExist_UpdateFunctionIsAppliedAsync() { - var componentRecorder = new ComponentRecorder(); + var componentRecorder = new ComponentRecorder(new Mock().Object); var npmDetector = new Mock(); var args = new BcdeArguments { diff --git a/test/Microsoft.ComponentDetection.Orchestrator.Tests/Services/DetectorListingCommandServiceTests.cs b/test/Microsoft.ComponentDetection.Orchestrator.Tests/Services/DetectorListingCommandServiceTests.cs index c307b394b..adcfa6922 100644 --- a/test/Microsoft.ComponentDetection.Orchestrator.Tests/Services/DetectorListingCommandServiceTests.cs +++ b/test/Microsoft.ComponentDetection.Orchestrator.Tests/Services/DetectorListingCommandServiceTests.cs @@ -1,10 +1,13 @@ namespace Microsoft.ComponentDetection.Orchestrator.Tests.Services; + +using System; using System.Collections.Generic; using System.Threading.Tasks; using FluentAssertions; using Microsoft.ComponentDetection.Contracts; using Microsoft.ComponentDetection.Orchestrator.ArgumentSets; using Microsoft.ComponentDetection.Orchestrator.Services; +using Microsoft.Extensions.Logging; using Microsoft.VisualStudio.TestTools.UnitTesting; using Moq; @@ -13,7 +16,7 @@ namespace Microsoft.ComponentDetection.Orchestrator.Tests.Services; [TestCategory("Governance/ComponentDetection")] public class DetectorListingCommandServiceTests { - private Mock loggerMock; + private Mock> loggerMock; private Mock> detectorsMock; private Mock componentDetector2Mock; private Mock componentDetector3Mock; @@ -26,7 +29,7 @@ public class DetectorListingCommandServiceTests [TestInitialize] public void InitializeTest() { - this.loggerMock = new Mock(); + this.loggerMock = new Mock>(); this.detectorsMock = new Mock>(); this.componentDetector2Mock = new Mock(); this.componentDetector3Mock = new Mock(); @@ -37,10 +40,23 @@ public void InitializeTest() this.loggerMock.Object); this.logOutput = new List(); - this.loggerMock.Setup(x => x.LogInfo(It.IsAny())).Callback(loggedString => - { - this.logOutput.Add(loggedString); - }); + this.loggerMock.Setup(x => x.Log( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + (Func)It.IsAny())) + .Callback(new InvocationAction(invocation => + { + var state = invocation.Arguments[2]; + var exception = (Exception)invocation.Arguments[3]; + var formatter = invocation.Arguments[4]; + + var invokeMethod = formatter.GetType().GetMethod("Invoke"); + var logMessage = (string)invokeMethod?.Invoke(formatter, new[] { state, exception }); + + this.logOutput.Add(logMessage); + })); this.componentDetector2Mock.SetupGet(x => x.Id).Returns("ComponentDetector2"); this.componentDetector3Mock.SetupGet(x => x.Id).Returns("ComponentDetector3"); diff --git a/test/Microsoft.ComponentDetection.Orchestrator.Tests/Services/DetectorProcessingServiceTests.cs b/test/Microsoft.ComponentDetection.Orchestrator.Tests/Services/DetectorProcessingServiceTests.cs index ce3fb97e8..675e339f8 100644 --- a/test/Microsoft.ComponentDetection.Orchestrator.Tests/Services/DetectorProcessingServiceTests.cs +++ b/test/Microsoft.ComponentDetection.Orchestrator.Tests/Services/DetectorProcessingServiceTests.cs @@ -11,6 +11,7 @@ namespace Microsoft.ComponentDetection.Orchestrator.Tests.Services; using Microsoft.ComponentDetection.Contracts.TypedComponent; using Microsoft.ComponentDetection.Orchestrator.ArgumentSets; using Microsoft.ComponentDetection.Orchestrator.Services; +using Microsoft.Extensions.Logging; using Microsoft.VisualStudio.TestTools.UnitTesting; using Moq; using Newtonsoft.Json; @@ -32,7 +33,7 @@ public class DetectorProcessingServiceTests { "experimentalFileDetectorId", new DetectedComponent(new NuGetComponent("experimentalDetectorName", "experimentalDetectorVersion")) }, }; - private readonly Mock loggerMock; + private readonly Mock> loggerMock; private readonly DetectorProcessingService serviceUnderTest; private readonly Mock directoryWalkerFactory; @@ -48,7 +49,7 @@ public class DetectorProcessingServiceTests public DetectorProcessingServiceTests() { - this.loggerMock = new Mock(); + this.loggerMock = new Mock>(); this.directoryWalkerFactory = new Mock(); this.serviceUnderTest = new DetectorProcessingService(this.directoryWalkerFactory.Object, this.loggerMock.Object); diff --git a/test/Microsoft.ComponentDetection.Orchestrator.Tests/Services/DetectorRestrictionServiceTests.cs b/test/Microsoft.ComponentDetection.Orchestrator.Tests/Services/DetectorRestrictionServiceTests.cs index 9b2ed51ec..cc86a158e 100644 --- a/test/Microsoft.ComponentDetection.Orchestrator.Tests/Services/DetectorRestrictionServiceTests.cs +++ b/test/Microsoft.ComponentDetection.Orchestrator.Tests/Services/DetectorRestrictionServiceTests.cs @@ -2,10 +2,10 @@ using System; using System.Linq; using FluentAssertions; -using Microsoft.ComponentDetection.Common; using Microsoft.ComponentDetection.Contracts; using Microsoft.ComponentDetection.Orchestrator.Exceptions; using Microsoft.ComponentDetection.Orchestrator.Services; +using Microsoft.Extensions.Logging; using Microsoft.VisualStudio.TestTools.UnitTesting; using Moq; @@ -14,7 +14,7 @@ [TestCategory("Governance/ComponentDetection")] public class DetectorRestrictionServiceTests { - private Mock logger; + private Mock> logger; private Mock firstDetectorMock; private Mock secondDetectorMock; private Mock thirdDetectorMock; @@ -26,7 +26,7 @@ public class DetectorRestrictionServiceTests [TestInitialize] public void TestInitialize() { - this.logger = new Mock(); + this.logger = new Mock>(); this.firstDetectorMock = this.GenerateDetector("FirstDetector"); this.secondDetectorMock = this.GenerateDetector("SecondDetector"); this.thirdDetectorMock = this.GenerateDetector("ThirdDetector"); diff --git a/test/Microsoft.ComponentDetection.TestsUtilities/DetectorTestUtilityBuilder.cs b/test/Microsoft.ComponentDetection.TestsUtilities/DetectorTestUtilityBuilder.cs index 3ef9100ed..9c3c353db 100644 --- a/test/Microsoft.ComponentDetection.TestsUtilities/DetectorTestUtilityBuilder.cs +++ b/test/Microsoft.ComponentDetection.TestsUtilities/DetectorTestUtilityBuilder.cs @@ -10,6 +10,7 @@ namespace Microsoft.ComponentDetection.TestsUtilities; using Microsoft.ComponentDetection.Contracts; using Microsoft.ComponentDetection.Contracts.Internal; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; using Moq; public class DetectorTestUtilityBuilder @@ -20,7 +21,7 @@ public class DetectorTestUtilityBuilder private readonly Mock mockComponentStreamEnumerableFactory; private readonly Mock mockObservableDirectoryWalkerFactory; - private readonly Mock mockLogger; + private readonly Mock> mockLogger; private readonly IServiceCollection serviceCollection; private T detector; @@ -40,7 +41,7 @@ public DetectorTestUtilityBuilder() this.serviceCollection.AddSingleton(_ => this.mockObservableDirectoryWalkerFactory?.Object); - this.mockLogger = new Mock(); + this.mockLogger = new Mock>(); this.serviceCollection.AddSingleton(_ => this.mockLogger?.Object); } diff --git a/test/Microsoft.ComponentDetection.TestsUtilities/Microsoft.ComponentDetection.TestsUtilities.csproj b/test/Microsoft.ComponentDetection.TestsUtilities/Microsoft.ComponentDetection.TestsUtilities.csproj index c97ad1e39..d9958a555 100644 --- a/test/Microsoft.ComponentDetection.TestsUtilities/Microsoft.ComponentDetection.TestsUtilities.csproj +++ b/test/Microsoft.ComponentDetection.TestsUtilities/Microsoft.ComponentDetection.TestsUtilities.csproj @@ -10,6 +10,7 @@ + diff --git a/test/Microsoft.ComponentDetection.VerificationTests/ComponentDetectionIntegrationTests.cs b/test/Microsoft.ComponentDetection.VerificationTests/ComponentDetectionIntegrationTests.cs index 43a5677e6..26ce6532e 100644 --- a/test/Microsoft.ComponentDetection.VerificationTests/ComponentDetectionIntegrationTests.cs +++ b/test/Microsoft.ComponentDetection.VerificationTests/ComponentDetectionIntegrationTests.cs @@ -178,7 +178,7 @@ public void CheckDetectorsRunTimesAndCounts() using (new AssertionScope()) { this.ProcessDetectorVersions(); - var regexPattern = @"Detection time: (\w+\.\w+) seconds. |(\w+ *[\w()]+) *\|(\w+\.*\w*) seconds *\|(\d+)"; + var regexPattern = @"Detection time: (\w+\.\w+) seconds.\w?|(\w+ *[\w()]+) *\|(\w+\.*\w*) seconds *\|(\d+)"; var oldMatches = Regex.Matches(this.oldLogFileContents, regexPattern); var newMatches = Regex.Matches(this.newLogFileContents, regexPattern); diff --git a/test/Microsoft.ComponentDetection.VerificationTests/Microsoft.DependencyDetective.VerificationTests.csproj b/test/Microsoft.ComponentDetection.VerificationTests/Microsoft.DependencyDetective.VerificationTests.csproj index 72bf23e1a..4e635aa46 100644 --- a/test/Microsoft.ComponentDetection.VerificationTests/Microsoft.DependencyDetective.VerificationTests.csproj +++ b/test/Microsoft.ComponentDetection.VerificationTests/Microsoft.DependencyDetective.VerificationTests.csproj @@ -16,14 +16,16 @@ IF TESTING LOCALLY, UNCOMMENT ABOVE AND COMMENT BELOW --> - - - - + + + + + + - + - \ No newline at end of file +