Skip to content
This repository has been archived by the owner on Sep 6, 2023. It is now read-only.

Commit

Permalink
Merge pull request #36 from LuccaSA/datadog-logs
Browse files Browse the repository at this point in the history 
Add audit log file
  • Loading branch information
@rducom
rducom authored Apr 12, 2022
2 parents 3f0713c + 4f07123 commit 0384d6c
Show file tree
Hide file tree
Showing 8 changed files with 94 additions and 20 deletions.
1 change: 1 addition & 0 deletions TCC.Lib/Command/CommandResult.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ public class CommandResult
public List<string> Infos { get; set; }
public long ElapsedMilliseconds => (long)Elapsed.TotalMilliseconds;
public TimeSpan Elapsed { get; set; }
public long ArchiveFileSize { get; set; }

public void ThrowOnError()
{
Expand Down
10 changes: 6 additions & 4 deletions TCC.Lib/OperationBlock.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ public interface IIterationResult
bool HasWarning { get; }
bool HasError { get; }

IEnumerable<StepResult> StepResults { get; }
List<StepResult> StepResults { get; }
}

public class OperationCompressionBlock : IIterationResult
Expand All @@ -38,7 +38,7 @@ public OperationCompressionBlock(CompressionBlock compressionBlock, CommandResul
public bool IsSuccess => BlockResult.StepResults.All(s => s.IsSuccess);
public bool HasWarning => BlockResult.StepResults.Any(i => i.HasWarning);
public bool HasError => BlockResult.StepResults.Any(i => i.HasError);
public IEnumerable<StepResult> StepResults => BlockResult.StepResults;
public List<StepResult> StepResults => BlockResult.StepResults;

public void ThrowOnError()
{
Expand Down Expand Up @@ -106,7 +106,7 @@ public bool IsSuccess

public bool HasWarning => BlockResults.Any(b => b.StepResults.Any(s => s.HasWarning));
public bool HasError => BlockResults.Any(b => b.StepResults.Any(s => s.HasError));
public IEnumerable<StepResult> StepResults => BlockResults.SelectMany(b => b.StepResults);
public List<StepResult> StepResults => BlockResults.SelectMany(b => b.StepResults).ToList();

public void ThrowOnError()
{
Expand Down Expand Up @@ -217,7 +217,8 @@ public BlockResult(Block block, CommandResult commandResult)
Type = StepType.Compression,
Duration = commandResult.Elapsed,
Infos = string.Join(Environment.NewLine, commandResult.Infos),
Errors = commandResult.Errors
Errors = commandResult.Errors,
ArchiveFileSize = commandResult.ArchiveFileSize
});
}

Expand All @@ -238,6 +239,7 @@ public class StepResult
public bool IsSuccess => !HasError && !HasWarning;
public bool HasError => !string.IsNullOrWhiteSpace(Errors);
public bool HasWarning => !string.IsNullOrWhiteSpace(Warning);
public long ArchiveFileSize { get; set; }

public void ThrowOnError()
{
Expand Down
1 change: 1 addition & 0 deletions TCC.Lib/Options/TccOption.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,5 +17,6 @@ public class TccOption

public bool Verbose { get; set; }
public string LogPaths { get; set; }
public string AuditFilePath { get; set; }
}
}
5 changes: 4 additions & 1 deletion TCC.Lib/TarCompressCrypt.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -120,7 +120,8 @@ private async Task<OperationCompressionBlock> UploadBlockInternal(IRemoteStorage
{
Type = StepType.Upload,
Errors = result.IsSuccess ? null : result.ErrorMessage,
Infos = result.IsSuccess ? result.ErrorMessage : null
Infos = result.IsSuccess ? result.ErrorMessage : null,
ArchiveFileSize = file.Length
});

sw.Stop();
Expand Down Expand Up @@ -260,6 +261,7 @@ private async Task<OperationCompressionBlock> CompressionBlockInternal(CompressO
{
string cmd = _compressionCommands.CompressCommand(block, option);
result = await cmd.Run(block.OperationFolder, token);
result.ArchiveFileSize = block.DestinationArchiveFileInfo.Length;
LogCompressionReport(block, result);

if (result.HasError)
Expand Down Expand Up @@ -394,6 +396,7 @@ private async Task<CommandResult> DecompressBlock(DecompressOption option, Decom
{
string cmd = _compressionCommands.DecompressCommand(block, option);
result = await cmd.Run(block.OperationFolder, token);
result.ArchiveFileSize = block.SourceArchiveFileInfo.Length;

LogReport(block, result);
}
Expand Down
4 changes: 3 additions & 1 deletion TCC/Parser/BaseCmdOptions.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,9 @@ public class BaseCmdOptions
[Option('v', "verbose", HelpText = "Verbose output", Default = false)]
public bool Verbose { get; set; }

[Option( "logPath", HelpText = "Log path")]
[Option("logPath", HelpText = "Log path")]
public string LogPaths { get; set; }
[Option("auditFile", HelpText = "Path to the audit log file")]
public string AuditFilePath { get; set; }
}
}
6 changes: 4 additions & 2 deletions TCC/Parser/ParseCommandLineHelper.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,8 @@ public static TccCommand ParseCommandLine(this string[] args)
GoogleStorageBucketName = opts.GoogleStorageBucketName,
GoogleStorageCredentialFile = opts.GoogleStorageCredentialFile,
AzThread = opts.AzThread,
UploadMode = opts.UploadMode
UploadMode = opts.UploadMode,
AuditFilePath = opts.AuditFilePath
};
ExtractPasswordInfo(opts, option, Mode.Compress);
Expand All @@ -73,7 +74,8 @@ public static TccCommand ParseCommandLine(this string[] args)
SlackChannel = opts.SlackChannel,
SlackSecret = opts.SlackSecret,
BucketName = opts.BucketName,
SlackOnlyOnError = opts.SlackOnlyOnError
SlackOnlyOnError = opts.SlackOnlyOnError,
AuditFilePath = opts.AuditFilePath
};
ExtractPasswordInfo(opts, option, Mode.Decompress);
Expand Down
85 changes: 74 additions & 11 deletions TCC/Program.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,10 @@
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using Serilog;
using Serilog.Context;
using Serilog.Core;
using Serilog.Events;
using Serilog.Formatting.Json;
using TCC.Lib;
using TCC.Lib.Benchmark;
using TCC.Lib.Blocks;
Expand All @@ -25,6 +27,7 @@ namespace TCC
public static class Program
{
private const string _tccMutex = "Global\\FD70BFC5-79C8-44DF-9629-65512A1CD0FC";
private const string AuditScope = "audit";

public static async Task<int> Main(string[] args)
{
Expand All @@ -41,7 +44,7 @@ public static async Task<int> Main(string[] args)
serviceCollection.AddTcc(workingPath);
serviceCollection.AddLogging(lb =>
{
var logger = CreateLogger(parsed.Mode, parsed.Option?.Verbose ?? false, parsed.Option?.LogPaths ?? workingPath);
var logger = CreateLogger(parsed.Mode, parsed.Option?.Verbose ?? false, parsed.Option?.LogPaths ?? workingPath, parsed.Option?.AuditFilePath);
lb.AddSerilog(logger, true);
});

Expand Down Expand Up @@ -77,6 +80,8 @@ public static async Task<int> Main(string[] args)

var notifier = sp.GetRequiredService<SlackSender>();
await notifier.ReportAsync(op, parsed.Option, parsed.Mode);

WriteAuditFile(parsed.Mode, op, logger);
}
}
catch (OperationCanceledException)
Expand Down Expand Up @@ -105,6 +110,27 @@ public static async Task<int> Main(string[] args)
return 0;
}

private static void WriteAuditFile(Mode mode, OperationSummary op, ILogger<TarCompressCrypt> logger)
{
using (logger.BeginScope(AuditScope))
{
foreach (IEnumerable<StepResult> stepResults in op.OperationBlocks.Select(o => o.StepResults).Where(result => result.Any()))
{
var name = stepResults.First().Name;
var logLevel = LogLevel.Information;
if (stepResults.Any(stepResult => stepResult.HasError))
{
logLevel = LogLevel.Error;
}
else if (stepResults.Any(stepResult => stepResult.HasWarning))
{
logLevel = LogLevel.Warning;
}
logger.Log(logLevel, "{Mode} / {Name} / {@results}", mode, name, stepResults);
}
}
}

private static IEnumerable<string> ReportOperationStats(OperationSummary op, Mode mode)
{
if (op == null)
Expand Down Expand Up @@ -194,11 +220,11 @@ private static async Task<OperationSummary> RunTcc(IServiceProvider provider, Tc
case Mode.Decompress:
var db = provider.GetRequiredService<DatabaseSetup>();
await db.EnsureDatabaseExistsAsync(command.Mode);

op = await provider
.GetRequiredService<TarCompressCrypt>()
.Decompress(command.Option as DecompressOption);

await db.CleanupDatabaseAsync(command.Mode);
break;
case Mode.Benchmark:
Expand All @@ -212,7 +238,7 @@ private static async Task<OperationSummary> RunTcc(IServiceProvider provider, Tc
return op;
}

private static Logger CreateLogger(Mode mode, bool verbose, string logDirectoryPath)
private static Logger CreateLogger(Mode mode, bool verbose, string logDirectoryPath, string auditFile)
{
string logFileName = mode switch
{
Expand Down Expand Up @@ -240,17 +266,54 @@ private static Logger CreateLogger(Mode mode, bool verbose, string logDirectoryP

var level = verbose ? LogEventLevel.Debug : LogEventLevel.Information;
var loggerConf = new LoggerConfiguration()
.MinimumLevel.Override("Microsoft.EntityFrameworkCore", LogEventLevel.Warning)
.WriteTo.Async(conf =>
.WriteTo.Logger(logger => logger
.MinimumLevel.Override("Microsoft.EntityFrameworkCore", LogEventLevel.Warning)
.WriteTo.Async(conf =>
{
if (logFileName != null)
{
conf.File(Path.Combine(path, logFileName), rollingInterval: RollingInterval.Day, retainedFileCountLimit: 31);
}
conf.Console();
})
.Filter.ByExcluding(IsAuditLog)
.MinimumLevel.Is(level)
);
if (!string.IsNullOrEmpty(auditFile))
{
if (!Path.IsPathRooted(auditFile))
{
if (logFileName != null)
auditFile = Path.Combine(path, auditFile);
}
loggerConf
.WriteTo.Logger(auditLogger =>
{
conf.File(Path.Combine(path, logFileName), rollingInterval: RollingInterval.Day, retainedFileCountLimit: 31);
}
conf.Console();
}).MinimumLevel.Is(level);
auditLogger
.MinimumLevel.Information()
.WriteTo.Async(conf =>
{
conf.File(new JsonFormatter(renderMessage: true), auditFile, rollingInterval: RollingInterval.Day, retainedFileCountLimit: 31);
})
.Filter.ByIncludingOnly(IsAuditLog);
});
}

return loggerConf.CreateLogger();
}

private static bool IsAuditLog(LogEvent logEvent)
{
LogEventPropertyValue scope = logEvent.Properties.GetValueOrDefault("Scope");
if (scope is null)
{
return false;
}
return
scope is SequenceValue sequenceValue &&
sequenceValue
.Elements
.OfType<ScalarValue>()
.Any(scalarValue => string.Equals(scalarValue.Value as string, AuditScope));
}
}
}
2 changes: 1 addition & 1 deletion TCC/TCC.csproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
<PackageReference Include="CommandLineParser" Version="2.8.0" />
<PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="6.0.0" />
<PackageReference Include="Microsoft.Extensions.Logging" Version="6.0.0" />
<PackageReference Include="Serilog.AspNetCore" Version="4.1.0" />
<PackageReference Include="Serilog.AspNetCore" Version="5.0.0" />
<PackageReference Include="Serilog.Sinks.Console" Version="4.0.1" />
<PackageReference Include="Serilog.Sinks.File" Version="5.0.0" />
</ItemGroup>
Expand Down

0 comments on commit 0384d6c

Please sign in to comment.