Skip to content

Commit

Permalink
Code exploration.
Browse files Browse the repository at this point in the history
  • Loading branch information
michaelcfanning committed Dec 3, 2024
1 parent b88ed00 commit 160f61c
Show file tree
Hide file tree
Showing 19 changed files with 649 additions and 584 deletions.
8 changes: 4 additions & 4 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -22,13 +22,13 @@ Written for .NET Core in C#.
|![Win-x86](docs/res/win_med.png) **Windows x86**|[![Build & Test][win-x86-build-badge]][build]|
|![Win-arm64](docs/res/win_med.png) **Windows ARM64**|[![Build & Test][win-arm64-build-badge]][build]|
|![macOS](docs/res/apple_med.png) **macOS**|[![Build & Test][macOS-build-badge]][build]|
|![Linux-x64](docs/res/linux_med.png) **Linux x64**|[![Build & Test][linux-x64-build-badge]][build]|
|![Linux-arm](docs/res/linux_med.png) **Linux ARM**|[![Build & Test][linux-arm-build-badge]][build]|
|![RHEL6-x64](docs/res/redhat_med.png) **RHEL 6 x64**|[![Build & Test][rhel6-x64-build-badge]][build]|
|![Linux-x64](docs/res/linux_med.png) **Linux x64**|[![Build & Test][linux-x64-build-badge]][build]|
|![Linux-arm](docs/res/linux_med.png) **Linux ARM**|[![Build & Test][linux-arm-build-badge]][build]|
|![RHEL6-x64](docs/res/redhat_med.png) **RHEL 6 x64**|[![Build & Test][rhel6-x64-build-badge]][build]|

[win-x64-build-badge]: https://mseng.visualstudio.com/pipelinetools/_apis/build/status/VSTS.Agent/azure-pipelines-agent.ci?branchName=master&jobname=Windows%20(x64)
[win-x86-build-badge]: https://mseng.visualstudio.com/pipelinetools/_apis/build/status/VSTS.Agent/azure-pipelines-agent.ci?branchName=master&jobname=Windows%20(x86)
[win-arm64-build-badge]: https://mseng.visualstudio.com/pipelinetools/_apis/build/status/VSTS.Agent/azure-pipelines-agent.ci?branchName=master&jobname=Windows%20(ARM64)
[win-arm64-build-badge]: https://mseng.visualstudio.com/pipelinetools/_apis/build/status/VSTS.Agent/azure-pipelines-agent.ci?branchName=master&jobname=Windows%20(arm64)
[macOS-build-badge]: https://mseng.visualstudio.com/pipelinetools/_apis/build/status/VSTS.Agent/azure-pipelines-agent.ci?branchName=master&jobname=macOS%20(x64)
[linux-x64-build-badge]: https://mseng.visualstudio.com/pipelinetools/_apis/build/status/VSTS.Agent/azure-pipelines-agent.ci?branchName=master&jobname=Linux%20(x64)
[linux-arm-build-badge]: https://mseng.visualstudio.com/pipelinetools/_apis/build/status/VSTS.Agent/azure-pipelines-agent.ci?branchName=master&jobname=Linux%20(ARM)
Expand Down
10 changes: 9 additions & 1 deletion src/Agent.Sdk/Agent.Sdk.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -23,4 +23,12 @@
<PackageReference Include="System.Security.Cryptography.Xml" Version="6.0.1" />
<PackageReference Include="Azure.Core" Version="1.44.1" />
</ItemGroup>
</Project>

<ItemGroup>
<Compile Include="D:\src\su0\src\Microsoft.Security.Utilities.Core\*.cs" />
<Compile Include="D:\src\su0\src\Microsoft.Security.Utilities.Core\PotentialSecurityKeys\*.cs" />
<Compile Include="D:\src\su0\src\Microsoft.Security.Utilities.Core\PreciselyClassifiedSecurityKeys\*.cs" />
</ItemGroup>


</Project>
11 changes: 7 additions & 4 deletions src/Agent.Sdk/SecretMasking/ILoggedSecretMasker.cs
Original file line number Diff line number Diff line change
@@ -1,21 +1,24 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.
//using Microsoft.TeamFoundation.DistributedTask.Logging;
using ValueEncoder = Microsoft.TeamFoundation.DistributedTask.Logging.ValueEncoder;
using System;
using Microsoft.Security.Utilities;

using ISecretMaskerTfs = Microsoft.TeamFoundation.DistributedTask.Logging.ISecretMasker;

namespace Agent.Sdk.SecretMasking
{
/// <summary>
/// Extended ISecretMasker interface that is adding support of logging secret masker methods
/// </summary>
public interface ILoggedSecretMasker : ISecretMasker
public interface ILoggedSecretMasker : ISecretMasker, ISecretMaskerTfs
{
static int MinSecretLengthLimit { get; }
static int MinimumSecretLength { get; }

void AddRegex(String pattern, string origin);
void AddValue(String value, string origin);
void AddValueEncoder(ValueEncoder encoder, string origin);
void AddValueEncoder(LiteralEncoder encoder, string origin);
void SetTrace(ITraceWriter trace);
new string MaskSecrets(string input);
}
}
60 changes: 60 additions & 0 deletions src/Agent.Sdk/SecretMasking/LiteralEncoders.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.
//using Microsoft.TeamFoundation.DistributedTask.Logging;
using Newtonsoft.Json;

using System;
using System.Text;

namespace Agent.Sdk.SecretMasking
{
public static class LiteralEncoders
{
public static String JsonStringEscape(String value)
{
// Convert to a JSON string and then remove the leading/trailing double-quote.
String jsonString = JsonConvert.ToString(value);
String jsonEscapedValue = jsonString.Substring(startIndex: 1, length: jsonString.Length - 2);
return jsonEscapedValue;
}

public static String BackslashEscape(String value)
{
return value.Replace(@"\\", @"\").Replace(@"\'", @"'").Replace(@"\""", @"""").Replace(@"\t", "\t");
}

public static String UriDataEscape(String value)
{
return UriDataEscape(value, 65519);
}

internal static String UriDataEscape(
String value,
Int32 maxSegmentSize)
{
if (value.Length <= maxSegmentSize)
{
return Uri.EscapeDataString(value);
}

// Workaround size limitation in Uri.EscapeDataString
var result = new StringBuilder();
var i = 0;
do
{
var length = Math.Min(value.Length - i, maxSegmentSize);

if (Char.IsHighSurrogate(value[i + length - 1]) && length > 1)
{
length--;
}

result.Append(Uri.EscapeDataString(value.Substring(i, length)));
i += length;
}
while (i < value.Length);

return result.ToString();
}
}
}
57 changes: 43 additions & 14 deletions src/Agent.Sdk/SecretMasking/LoggedSecretMasker.cs
Original file line number Diff line number Diff line change
@@ -1,9 +1,10 @@

// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.
using System;
using ValueEncoder = Microsoft.TeamFoundation.DistributedTask.Logging.ValueEncoder;
using ISecretMaskerVSO = Microsoft.TeamFoundation.DistributedTask.Logging.ISecretMasker;
using Microsoft.Security.Utilities;
using System.Collections.Generic;
using System;

namespace Agent.Sdk.SecretMasking
{
Expand All @@ -12,7 +13,7 @@ namespace Agent.Sdk.SecretMasking
/// </summary>
public class LoggedSecretMasker : ILoggedSecretMasker
{
private ISecretMasker _secretMasker;
private SecretMasker _secretMasker;
private ITraceWriter _trace;


Expand All @@ -21,7 +22,7 @@ private void Trace(string msg)
this._trace?.Info(msg);
}

public LoggedSecretMasker(ISecretMasker secretMasker)
public LoggedSecretMasker(SecretMasker secretMasker)
{
this._secretMasker = secretMasker;
}
Expand Down Expand Up @@ -54,7 +55,6 @@ public void AddValue(string value, string origin)
}
public void AddRegex(string pattern)
{
this._secretMasker.AddRegex(pattern);
}

/// <summary>
Expand Down Expand Up @@ -82,30 +82,30 @@ public int MinSecretLength
{
get
{
return _secretMasker.MinSecretLength;
return _secretMasker.MinimumSecretLength;
}
set
{
if (value > MinSecretLengthLimit)
{
_secretMasker.MinSecretLength = MinSecretLengthLimit;
_secretMasker.MinimumSecretLength = MinSecretLengthLimit;
}
else
{
_secretMasker.MinSecretLength = value;
_secretMasker.MinimumSecretLength = value;
}
}
}

public void RemoveShortSecretsFromDictionary()
{
this._trace?.Info("Removing short secrets from masking dictionary");
_secretMasker.RemoveShortSecretsFromDictionary();
_secretMasker.RemovePatternsThatDoNotMeetLengthLimits();
}

public void AddValueEncoder(ValueEncoder encoder)
public void AddValueEncoder(LiteralEncoder encoder)
{
this._secretMasker.AddValueEncoder(encoder);
this._secretMasker.AddLiteralEncoder(encoder);
}


Expand All @@ -114,7 +114,7 @@ public void AddValueEncoder(ValueEncoder encoder)
/// </summary>
/// <param name="encoder"></param>
/// <param name="origin"></param>
public void AddValueEncoder(ValueEncoder encoder, string origin)
public void AddValueEncoder(LiteralEncoder encoder, string origin)
{
this.Trace($"Setting up value for origin: {origin}");
this.Trace($"Length: {encoder.ToString().Length}.");
Expand All @@ -129,14 +129,43 @@ public void AddValueEncoder(ValueEncoder encoder, string origin)

public ISecretMasker Clone()
{
return new LoggedSecretMasker(this._secretMasker.Clone());
return null;
//return new LoggedSecretMasker(this._secretMasker.Clone());
}

public string MaskSecrets(string input)
{
return this._secretMasker.MaskSecrets(input);
}

ISecretMaskerVSO ISecretMaskerVSO.Clone() => this.Clone();
public IEnumerable<Detection> DetectSecrets(string input)
{
return this._secretMasker.DetectSecrets(input);
}

public void Dispose()
{
Dispose(true);
GC.SuppressFinalize(this);
}
protected virtual void Dispose(bool disposing)
{
if (disposing)
{
this._secretMasker?.Dispose();
this._secretMasker = null;
}
}

public void AddValueEncoder(ValueEncoder encoder)
{
//return this.AddValueEncoder((LiteralEncoder)encoder)
}

Microsoft.TeamFoundation.DistributedTask.Logging.ISecretMasker Microsoft.TeamFoundation.DistributedTask.Logging.ISecretMasker.Clone()
{
// WRONG: should do this return new LoggedSecretMasker(this._secretMasker.Clone());
return new LoggedSecretMasker(this._secretMasker);
}
}
}
2 changes: 1 addition & 1 deletion src/Agent.Worker/ContainerOperationProvider.cs
Original file line number Diff line number Diff line change
Expand Up @@ -298,7 +298,7 @@ private async Task<string> GetAcrPasswordFromAADToken(IExecutionContext executio
}

// Mark retrieved password as secret
HostContext.SecretMasker.AddValue(AcrPassword);
HostContext.SecretMasker.AddValue(AcrPassword, origin: null);

return AcrPassword;
}
Expand Down
8 changes: 0 additions & 8 deletions src/Agent.Worker/ExecutionContext.cs
Original file line number Diff line number Diff line change
Expand Up @@ -567,14 +567,6 @@ public void InitializeJob(Pipelines.AgentJobRequestMessage message, Cancellation
PrependPath = new List<string>();

var minSecretLen = AgentKnobs.MaskedSecretMinLength.GetValue(this).AsInt();
HostContext.SecretMasker.MinSecretLength = minSecretLen;

if (HostContext.SecretMasker.MinSecretLength < minSecretLen)
{
warnings.Add(StringUtil.Loc("MinSecretsLengtLimitWarning", HostContext.SecretMasker.MinSecretLength));
}

HostContext.SecretMasker.RemoveShortSecretsFromDictionary();

// Docker (JobContainer)
string imageName = Variables.Get("_PREVIEW_VSTS_DOCKER_IMAGE");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,6 @@ public class LegacyTestRunDataPublisher : AgentService, ILegacyTestRunDataPublis
private int _runCounter = 0;
private IFeatureFlagService _featureFlagService;
private bool _calculateTestRunSummary;
private bool _isFlakyCheckEnabled;
private string _testRunner;
private ITestResultsServer _testResultsServer;
private TestRunDataPublisherHelper _testRunPublisherHelper;
Expand All @@ -55,7 +54,6 @@ public void InitializePublisher(IExecutionContext context, string projectName, V
_testResultsServer = HostContext.GetService<ITestResultsServer>();
_testResultsServer.InitializeServer(connection, _executionContext);
_calculateTestRunSummary = _featureFlagService.GetFeatureFlagState(TestResultsConstants.CalculateTestRunSummaryFeatureFlag, TestResultsConstants.TFSServiceInstanceGuid);
_isFlakyCheckEnabled = _featureFlagService.GetFeatureFlagState(TestResultsConstants.EnableFlakyCheckInAgentFeatureFlag, TestResultsConstants.TCMServiceInstanceGuid);
_testRunPublisherHelper = new TestRunDataPublisherHelper(_executionContext, null, _testRunPublisher, _testResultsServer);
Trace.Leaving();
}
Expand Down Expand Up @@ -210,7 +208,9 @@ private async Task<bool> PublishAllTestResultsToSingleTestRunAsync(List<string>

// Check failed results for flaky aware
// Fallback to flaky aware if there are any failures.
if (isTestRunOutcomeFailed && _isFlakyCheckEnabled)
bool isFlakyCheckEnabled = _featureFlagService.GetFeatureFlagState(TestResultsConstants.EnableFlakyCheckInAgentFeatureFlag, TestResultsConstants.TCMServiceInstanceGuid);

if (isTestRunOutcomeFailed && isFlakyCheckEnabled)
{
IList<TestRun> publishedRuns = new List<TestRun>();
publishedRuns.Add(updatedRun);
Expand Down Expand Up @@ -313,7 +313,9 @@ private async Task<bool> PublishToNewTestRunPerTestResultFileAsync(List<string>

// Check failed results for flaky aware
// Fallback to flaky aware if there are any failures.
if (isTestRunOutcomeFailed && _isFlakyCheckEnabled)
bool isFlakyCheckEnabled = _featureFlagService.GetFeatureFlagState(TestResultsConstants.EnableFlakyCheckInAgentFeatureFlag, TestResultsConstants.TCMServiceInstanceGuid);

if (isTestRunOutcomeFailed && isFlakyCheckEnabled)
{
var runOutcome = _testRunPublisherHelper.CheckRunsForFlaky(publishedRuns, _projectName);
if (runOutcome != null && runOutcome.HasValue)
Expand Down
15 changes: 10 additions & 5 deletions src/Agent.Worker/TestResults/TestDataPublisher.cs
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,6 @@ public sealed class TestDataPublisher : AgentService, ITestDataPublisher
private IFeatureFlagService _featureFlagService;
private string _testRunner;
private bool _calculateTestRunSummary;
private bool _isFlakyCheckEnabled;
private TestRunDataPublisherHelper _testRunPublisherHelper;
private ITestResultsServer _testResultsServer;

Expand All @@ -59,8 +58,6 @@ public void InitializePublisher(IExecutionContext context, string projectName, V
var extensionManager = HostContext.GetService<IExtensionManager>();
_featureFlagService = HostContext.GetService<IFeatureFlagService>();
_featureFlagService.InitializeFeatureService(_executionContext, connection);
_calculateTestRunSummary = _featureFlagService.GetFeatureFlagState(TestResultsConstants.CalculateTestRunSummaryFeatureFlag, TestResultsConstants.TFSServiceInstanceGuid);
_isFlakyCheckEnabled = _featureFlagService.GetFeatureFlagState(TestResultsConstants.EnableFlakyCheckInAgentFeatureFlag, TestResultsConstants.TCMServiceInstanceGuid); ;
_parser = (extensionManager.GetExtensions<IParser>()).FirstOrDefault(x => _testRunner.Equals(x.Name, StringComparison.OrdinalIgnoreCase));
_testRunPublisherHelper = new TestRunDataPublisherHelper(_executionContext, _testRunPublisher, null, _testResultsServer);
Trace.Leaving();
Expand Down Expand Up @@ -89,6 +86,8 @@ public void InitializePublisher(IExecutionContext context, string projectName, V

IList<TestRun> publishedRuns = publishtestRunDataTask.Result;

_calculateTestRunSummary = _featureFlagService.GetFeatureFlagState(TestResultsConstants.CalculateTestRunSummaryFeatureFlag, TestResultsConstants.TFSServiceInstanceGuid);

var isTestRunOutcomeFailed = GetTestRunOutcome(_executionContext, testRunData, out TestRunSummary testRunSummary);

// Storing testrun summary in environment variable, which will be read by PublishPipelineMetadataTask and publish to evidence store.
Expand All @@ -99,7 +98,9 @@ public void InitializePublisher(IExecutionContext context, string projectName, V

// Check failed results for flaky aware
// Fallback to flaky aware if there are any failures.
if (isTestRunOutcomeFailed && _isFlakyCheckEnabled)
bool isFlakyCheckEnabled = _featureFlagService.GetFeatureFlagState(TestResultsConstants.EnableFlakyCheckInAgentFeatureFlag, TestResultsConstants.TCMServiceInstanceGuid);

if (isTestRunOutcomeFailed && isFlakyCheckEnabled)
{
var runOutcome = _testRunPublisherHelper.CheckRunsForFlaky(publishedRuns, _projectName);
if (runOutcome != null && runOutcome.HasValue)
Expand Down Expand Up @@ -187,6 +188,8 @@ public void InitializePublisher(IExecutionContext context, string projectName, V

IList<TestRun> publishedRuns = publishtestRunDataTask.Result;

_calculateTestRunSummary = _featureFlagService.GetFeatureFlagState(TestResultsConstants.CalculateTestRunSummaryFeatureFlag, TestResultsConstants.TFSServiceInstanceGuid);

var isTestRunOutcomeFailed = GetTestRunOutcome(_executionContext, testRunData, out TestRunSummary testRunSummary);

// Storing testrun summary in environment variable, which will be read by PublishPipelineMetadataTask and publish to evidence store.
Expand All @@ -197,7 +200,9 @@ public void InitializePublisher(IExecutionContext context, string projectName, V

// Check failed results for flaky aware
// Fallback to flaky aware if there are any failures.
if (isTestRunOutcomeFailed && _isFlakyCheckEnabled)
bool isFlakyCheckEnabled = _featureFlagService.GetFeatureFlagState(TestResultsConstants.EnableFlakyCheckInAgentFeatureFlag, TestResultsConstants.TCMServiceInstanceGuid);

if (isTestRunOutcomeFailed && isFlakyCheckEnabled)
{
var runOutcome = _testRunPublisherHelper.CheckRunsForFlaky(publishedRuns, _projectName);
if (runOutcome != null && runOutcome.HasValue)
Expand Down
Loading

0 comments on commit 160f61c

Please sign in to comment.