Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add capability to publish/download pipeline artifact in a different domain. #4482

Merged
merged 15 commits into from
Oct 24, 2023
Merged

Add capability to publish/download pipeline artifact in a different domain. #4482

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
281ff9e
Initial pass at adding domain support for Pipeline Artifacts
Aug 4, 2023
42669db
Change missed domainId and some refactor
fadnavistanmay Sep 6, 2023
5139251
It works
fadnavistanmay Oct 3, 2023
68040d2
Merge branch 'master' into users/tafadnav/pipelineartdomain
fadnavistanmay Oct 3, 2023
c8cdd8b
Saving company's money by using loc string
fadnavistanmay Oct 9, 2023
5288f7e
Merge branch 'master' into users/tafadnav/pipelineartdomain
fadnavistanmay Oct 10, 2023
56a6d4b
Merge branch 'master' into users/tafadnav/pipelineartdomain
fadnavistanmay Oct 12, 2023
ce97060
Merge branch 'master' into users/tafadnav/pipelineartdomain
fadnavistanmay Oct 12, 2023
b14e7c4
Merge branch 'master' into users/tafadnav/pipelineartdomain
fadnavistanmay Oct 19, 2023
15778e2
Fix bug, use pipeline artifact instead of artifact
fadnavistanmay Oct 18, 2023
1642547
Merge branch 'users/tafadnav/pipelineartdomain' of https://github.com…
fadnavistanmay Oct 19, 2023
e2bd8e2
Add custom minimatch options
fadnavistanmay Oct 19, 2023
ecc3112
Merge branch 'master' into users/tafadnav/pipelineartdomain
fadnavistanmay Oct 20, 2023
f7d15c7
Merge branch 'master' into users/tafadnav/pipelineartdomain
fadnavistanmay Oct 20, 2023
f1d47b0
Merge branch 'master' into users/tafadnav/pipelineartdomain
fadnavistanmay Oct 24, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions src/Agent.Plugins/Artifact/PipelineArtifactConstants.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@

namespace Agent.Plugins
{
// Use PipelineArtifactContants.cs from ADO, once the latest libs are available.
public class PipelineArtifactConstants
{
public const string AzurePipelinesAgent = "AzurePipelinesAgent";
Expand All @@ -18,5 +19,6 @@ public class PipelineArtifactConstants
public const string FileShareArtifact = "filepath";
public const string CustomPropertiesPrefix = "user-";
public const string HashType = "HashType";
public const string DomainId = "DomainId";
}
}
126 changes: 84 additions & 42 deletions src/Agent.Plugins/Artifact/PipelineArtifactProvider.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
using Microsoft.VisualStudio.Services.WebApi;
using Microsoft.VisualStudio.Services.Content.Common;
using Microsoft.VisualStudio.Services.BlobStore.Common;
using Microsoft.VisualStudio.Services.BlobStore.Common.Telemetry;

namespace Agent.Plugins
{
Expand All @@ -37,12 +38,19 @@ public async Task DownloadSingleArtifactAsync(
CancellationToken cancellationToken,
AgentTaskPluginExecutionContext context)
{
// if properties doesn't have it, use the default domain for backward compatibility
IDomainId domainId = WellKnownDomainIds.DefaultDomainId;
if(buildArtifact.Resource.Properties.TryGetValue(PipelineArtifactConstants.DomainId, out string domainIdString))
{
domainId = DomainIdFactory.Create(domainIdString);
}

var (dedupManifestClient, clientTelemetry) = await DedupManifestArtifactClientFactory.Instance.CreateDedupManifestClientAsync(
this.context.IsSystemDebugTrue(),
(str) => this.context.Output(str),
this.connection,
DedupManifestArtifactClientFactory.Instance.GetDedupStoreClientMaxParallelism(context),
WellKnownDomainIds.DefaultDomainId,
domainId,
Microsoft.VisualStudio.Services.BlobStore.WebApi.Contracts.Client.PipelineArtifact,
context,
cancellationToken);
Expand All @@ -54,7 +62,8 @@ public async Task DownloadSingleArtifactAsync(
manifestId,
downloadParameters.TargetDirectory,
proxyUri: null,
minimatchPatterns: downloadParameters.MinimatchFilters);
minimatchPatterns: downloadParameters.MinimatchFilters,
customMinimatchOptions: downloadParameters.CustomMinimatchOptions);

PipelineArtifactActionRecord downloadRecord = clientTelemetry.CreateRecord<PipelineArtifactActionRecord>((level, uri, type) =>
new PipelineArtifactActionRecord(level, uri, type, nameof(DownloadMultipleArtifactsAsync), this.context));
Expand Down Expand Up @@ -85,49 +94,82 @@ public async Task DownloadMultipleArtifactsAsync(
CancellationToken cancellationToken,
AgentTaskPluginExecutionContext context)
{
var (dedupManifestClient, clientTelemetry) = await DedupManifestArtifactClientFactory.Instance.CreateDedupManifestClientAsync(
this.context.IsSystemDebugTrue(),
(str) => this.context.Output(str),
this.connection,
DedupManifestArtifactClientFactory.Instance.GetDedupStoreClientMaxParallelism(context),
WellKnownDomainIds.DefaultDomainId,
Microsoft.VisualStudio.Services.BlobStore.WebApi.Contracts.Client.PipelineArtifact,
context,
cancellationToken);
// create clients and group artifacts for each domain:
Dictionary<IDomainId, (DedupManifestArtifactClient Client, BlobStoreClientTelemetry Telemetry, Dictionary<string, DedupIdentifier> ArtifactDictionary)> dedupManifestClients =
new();

using (clientTelemetry)
{
var artifactNameAndManifestIds = buildArtifacts.ToDictionary(
keySelector: (a) => a.Name, // keys should be unique, if not something is really wrong
elementSelector: (a) => DedupIdentifier.Create(a.Resource.Data));
// 2) download to the target path
var options = DownloadDedupManifestArtifactOptions.CreateWithMultiManifestIds(
artifactNameAndManifestIds,
downloadParameters.TargetDirectory,
proxyUri: null,
minimatchPatterns: downloadParameters.MinimatchFilters,
minimatchFilterWithArtifactName: downloadParameters.MinimatchFilterWithArtifactName);
foreach(var buildArtifact in buildArtifacts)
{
// if properties doesn't have it, use the default domain for backward compatibility
IDomainId domainId = WellKnownDomainIds.DefaultDomainId;
if(buildArtifact.Resource.Properties.TryGetValue(PipelineArtifactConstants.DomainId, out string domainIdString))
{
domainId = DomainIdFactory.Create(domainIdString);
}

PipelineArtifactActionRecord downloadRecord = clientTelemetry.CreateRecord<PipelineArtifactActionRecord>((level, uri, type) =>
new PipelineArtifactActionRecord(level, uri, type, nameof(DownloadMultipleArtifactsAsync), this.context));
await clientTelemetry.MeasureActionAsync(
record: downloadRecord,
actionAsync: async () =>
// Have we already created the clients for this domain?
if(dedupManifestClients.ContainsKey(domainId)) {
// Clients already created for this domain, Just add the artifact to the list:
dedupManifestClients[domainId].ArtifactDictionary.Add(buildArtifact.Name, DedupIdentifier.Create(buildArtifact.Resource.Data));
}
else
{
// create the clients:
var (dedupManifestClient, clientTelemetry) = await DedupManifestArtifactClientFactory.Instance.CreateDedupManifestClientAsync(
this.context.IsSystemDebugTrue(),
(str) => this.context.Output(str),
this.connection,
DedupManifestArtifactClientFactory.Instance.GetDedupStoreClientMaxParallelism(context),
domainId,
Microsoft.VisualStudio.Services.BlobStore.WebApi.Contracts.Client.PipelineArtifact,
context,
cancellationToken);

// and create the artifact dictionary with the current artifact
var artifactDictionary = new Dictionary<string, DedupIdentifier>
{
await AsyncHttpRetryHelper.InvokeVoidAsync(
async () =>
{
await dedupManifestClient.DownloadAsync(options, cancellationToken);
},
maxRetries: 3,
tracer: tracer,
canRetryDelegate: e => true,
context: nameof(DownloadMultipleArtifactsAsync),
cancellationToken: cancellationToken,
continueOnCapturedContext: false);
});
// Send results to CustomerIntelligence
this.context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineArtifact, record: downloadRecord);
{ buildArtifact.Name, DedupIdentifier.Create(buildArtifact.Resource.Data) }
};

dedupManifestClients.Add(domainId, (dedupManifestClient, clientTelemetry, artifactDictionary));
}
}

foreach(var clientInfo in dedupManifestClients.Values)
{
using (clientInfo.Telemetry)
{
// 2) download to the target path
var options = DownloadDedupManifestArtifactOptions.CreateWithMultiManifestIds(
clientInfo.ArtifactDictionary,
downloadParameters.TargetDirectory,
proxyUri: null,
minimatchPatterns: downloadParameters.MinimatchFilters,
minimatchFilterWithArtifactName: downloadParameters.MinimatchFilterWithArtifactName,
customMinimatchOptions: downloadParameters.CustomMinimatchOptions);

PipelineArtifactActionRecord downloadRecord = clientInfo.Telemetry.CreateRecord<PipelineArtifactActionRecord>((level, uri, type) =>
new PipelineArtifactActionRecord(level, uri, type, nameof(DownloadMultipleArtifactsAsync), this.context));

await clientInfo.Telemetry.MeasureActionAsync(
record: downloadRecord,
actionAsync: async () =>
{
await AsyncHttpRetryHelper.InvokeVoidAsync(
async () =>
{
await clientInfo.Client.DownloadAsync(options, cancellationToken);
},
maxRetries: 3,
tracer: tracer,
canRetryDelegate: e => true,
context: nameof(DownloadMultipleArtifactsAsync),
cancellationToken: cancellationToken,
continueOnCapturedContext: false);
});
// Send results to CustomerIntelligence
this.context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineArtifact, record: downloadRecord);
}
}
}
}
Expand Down
102 changes: 20 additions & 82 deletions src/Agent.Plugins/Artifact/PipelineArtifactServer.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,15 +41,26 @@ internal async Task UploadAsync(
IDictionary<string, string> properties,
CancellationToken cancellationToken)
{
// Get the client settings, if any.
var tracer = DedupManifestArtifactClientFactory.CreateArtifactsTracer(verbose: false, (str) => context.Output(str));
VssConnection connection = context.VssConnection;
var (dedupManifestClient, clientTelemetry) = await DedupManifestArtifactClientFactory.Instance
.CreateDedupManifestClientAsync(
var clientSettings = await DedupManifestArtifactClientFactory.GetClientSettingsAsync(
connection,
Microsoft.VisualStudio.Services.BlobStore.WebApi.Contracts.Client.PipelineArtifact,
tracer,
cancellationToken);

// Get the default domain to use:
IDomainId domainId = DedupManifestArtifactClientFactory.GetDefaultDomainId(clientSettings, tracer);

var (dedupManifestClient, clientTelemetry) = DedupManifestArtifactClientFactory.Instance
.CreateDedupManifestClient(
context.IsSystemDebugTrue(),
(str) => context.Output(str),
connection,
DedupManifestArtifactClientFactory.Instance.GetDedupStoreClientMaxParallelism(context),
WellKnownDomainIds.DefaultDomainId,
Microsoft.VisualStudio.Services.BlobStore.WebApi.Contracts.Client.PipelineArtifact,
domainId,
clientSettings,
context,
cancellationToken);

Expand Down Expand Up @@ -84,7 +95,8 @@ internal async Task UploadAsync(
{ PipelineArtifactConstants.RootId, result.RootId.ValueString },
{ PipelineArtifactConstants.ProofNodes, StringUtil.ConvertToJson(result.ProofNodes.ToArray()) },
{ PipelineArtifactConstants.ArtifactSize, result.ContentSize.ToString() },
{ PipelineArtifactConstants.HashType, dedupManifestClient.HashType.Serialize() }
{ PipelineArtifactConstants.HashType, dedupManifestClient.HashType.Serialize() },
{ PipelineArtifactConstants.DomainId, domainId.Serialize() }
};

BuildArtifact buildArtifact = await AsyncHttpRetryHelper.InvokeAsync(
Expand Down Expand Up @@ -140,22 +152,11 @@ internal async Task DownloadAsync(
CancellationToken cancellationToken)
{
VssConnection connection = context.VssConnection;
var (dedupManifestClient, clientTelemetry) = await DedupManifestArtifactClientFactory.Instance
.CreateDedupManifestClientAsync(
context.IsSystemDebugTrue(),
(str) => context.Output(str),
connection,
DedupManifestArtifactClientFactory.Instance.GetDedupStoreClientMaxParallelism(context),
WellKnownDomainIds.DefaultDomainId,
Microsoft.VisualStudio.Services.BlobStore.WebApi.Contracts.Client.PipelineArtifact,
context,
cancellationToken);
PipelineArtifactProvider provider = new PipelineArtifactProvider(context, connection, tracer);

BuildServer buildServer = new(connection);

using (clientTelemetry)
// download all pipeline artifacts if artifact name is missing
{
if (downloadOptions == DownloadOptions.MultiDownload)
{
List<BuildArtifact> artifacts;
Expand Down Expand Up @@ -187,40 +188,7 @@ internal async Task DownloadAsync(
else
{
context.Output(StringUtil.Loc("DownloadingMultiplePipelineArtifacts", pipelineArtifacts.Count()));

var artifactNameAndManifestIds = pipelineArtifacts.ToDictionary(
keySelector: (a) => a.Name, // keys should be unique, if not something is really wrong
elementSelector: (a) => DedupIdentifier.Create(a.Resource.Data));
// 2) download to the target path
var options = DownloadDedupManifestArtifactOptions.CreateWithMultiManifestIds(
artifactNameAndManifestIds,
downloadParameters.TargetDirectory,
proxyUri: null,
minimatchPatterns: downloadParameters.MinimatchFilters,
minimatchFilterWithArtifactName: downloadParameters.MinimatchFilterWithArtifactName,
customMinimatchOptions: downloadParameters.CustomMinimatchOptions);

PipelineArtifactActionRecord downloadRecord = clientTelemetry.CreateRecord<PipelineArtifactActionRecord>((level, uri, type) =>
new PipelineArtifactActionRecord(level, uri, type, nameof(DownloadAsync), context));
await clientTelemetry.MeasureActionAsync(
record: downloadRecord,
actionAsync: async () =>
{
await AsyncHttpRetryHelper.InvokeVoidAsync(
async () =>
{
await dedupManifestClient.DownloadAsync(options, cancellationToken);
},
maxRetries: 3,
tracer: tracer,
canRetryDelegate: e => true,
context: nameof(DownloadAsync),
cancellationToken: cancellationToken,
continueOnCapturedContext: false);
});

// Send results to CustomerIntelligence
context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineArtifact, record: downloadRecord);
await provider.DownloadMultipleArtifactsAsync(downloadParameters, pipelineArtifacts, cancellationToken, context);
}
}
else if (downloadOptions == DownloadOptions.SingleDownload)
Expand All @@ -246,42 +214,12 @@ await AsyncHttpRetryHelper.InvokeVoidAsync(
{
throw new InvalidOperationException($"Invalid {nameof(downloadParameters.ProjectRetrievalOptions)}!");
}

var manifestId = DedupIdentifier.Create(buildArtifact.Resource.Data);
var options = DownloadDedupManifestArtifactOptions.CreateWithManifestId(
manifestId,
downloadParameters.TargetDirectory,
proxyUri: null,
minimatchPatterns: downloadParameters.MinimatchFilters,
customMinimatchOptions: downloadParameters.CustomMinimatchOptions);

PipelineArtifactActionRecord downloadRecord = clientTelemetry.CreateRecord<PipelineArtifactActionRecord>((level, uri, type) =>
new PipelineArtifactActionRecord(level, uri, type, nameof(DownloadAsync), context));
await clientTelemetry.MeasureActionAsync(
record: downloadRecord,
actionAsync: async () =>
{
await AsyncHttpRetryHelper.InvokeVoidAsync(
async () =>
{
await dedupManifestClient.DownloadAsync(options, cancellationToken);
},
maxRetries: 3,
tracer: tracer,
canRetryDelegate: e => true,
context: nameof(DownloadAsync),
cancellationToken: cancellationToken,
continueOnCapturedContext: false);
});

// Send results to CustomerIntelligence
context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineArtifact, record: downloadRecord);
await provider.DownloadSingleArtifactAsync(downloadParameters, buildArtifact, cancellationToken, context);
}
else
{
throw new InvalidOperationException($"Invalid {nameof(downloadOptions)}!");
}
}
}

// Download for version 2. This decision was made because version 1 is sealed and we didn't want to break any existing customers.
Expand Down
Loading
Loading