From a54d86b7cdd09f6d62290e78fd00201434317fda Mon Sep 17 00:00:00 2001 From: jnilau <109766438+jnilau@users.noreply.github.com> Date: Wed, 16 Oct 2024 11:43:38 -0700 Subject: [PATCH] Support sparse checkout (#5015) * Handle sparse checkout YAML properties * Fail task in case of issues that a user needs to address e.g. pattern format --- src/Agent.Plugins/GitCliManager.cs | 34 ++++++++++++++++++++++++++ src/Agent.Plugins/GitSourceProvider.cs | 29 ++++++++++++++++++++++ src/Agent.Sdk/Knob/AgentKnobs.cs | 6 +++++ src/Common.props | 2 +- 4 files changed, 70 insertions(+), 1 deletion(-) diff --git a/src/Agent.Plugins/GitCliManager.cs b/src/Agent.Plugins/GitCliManager.cs index 258d820764..b3a9d72a54 100644 --- a/src/Agent.Plugins/GitCliManager.cs +++ b/src/Agent.Plugins/GitCliManager.cs @@ -320,6 +320,40 @@ public async Task GitLFSFetch(AgentTaskPluginExecutionContext context, stri return fetchExitCode; } + // git sparse-checkout + public async Task GitSparseCheckout(AgentTaskPluginExecutionContext context, string repositoryPath, string directories, string patterns, CancellationToken cancellationToken) + { + context.Debug($"Sparse checkout"); + + bool useConeMode = !string.IsNullOrWhiteSpace(directories); + string options = useConeMode ? "--cone" : "--no-cone"; + + context.PublishTelemetry(area: "AzurePipelinesAgent", feature: "GitSparseCheckout", properties: new Dictionary + { + { "Mode", useConeMode ? "cone" : "non-cone" }, + { "Patterns", useConeMode ? directories : patterns } + }); + + int exitCode_sparseCheckoutInit = await ExecuteGitCommandAsync(context, repositoryPath, "sparse-checkout init", options, cancellationToken); + + if (exitCode_sparseCheckoutInit != 0) + { + return exitCode_sparseCheckoutInit; + } + else + { + return await ExecuteGitCommandAsync(context, repositoryPath, "sparse-checkout set", useConeMode ? directories : patterns, cancellationToken); + } + } + + // git sparse-checkout disable + public async Task GitSparseCheckoutDisable(AgentTaskPluginExecutionContext context, string repositoryPath, CancellationToken cancellationToken) + { + context.Debug($"Sparse checkout disable"); + + return await ExecuteGitCommandAsync(context, repositoryPath, "sparse-checkout disable", string.Empty, cancellationToken); + } + // git checkout -f --progress public async Task GitCheckout(AgentTaskPluginExecutionContext context, string repositoryPath, string committishOrBranchSpec, string additionalCommandLine, CancellationToken cancellationToken) { diff --git a/src/Agent.Plugins/GitSourceProvider.cs b/src/Agent.Plugins/GitSourceProvider.cs index 9e7123fe57..b8144bdf52 100644 --- a/src/Agent.Plugins/GitSourceProvider.cs +++ b/src/Agent.Plugins/GitSourceProvider.cs @@ -342,6 +342,10 @@ public async Task GetSourceAsync( string fetchFilter = executionContext.GetInput(Pipelines.PipelineConstants.CheckoutTaskInputs.FetchFilter); + string sparseCheckoutDirectories = executionContext.GetInput(Pipelines.PipelineConstants.CheckoutTaskInputs.SparseCheckoutDirectories); + string sparseCheckoutPatterns = executionContext.GetInput(Pipelines.PipelineConstants.CheckoutTaskInputs.SparseCheckoutPatterns); + bool enableSparseCheckout = !string.IsNullOrWhiteSpace(sparseCheckoutDirectories) || !string.IsNullOrWhiteSpace(sparseCheckoutPatterns); + executionContext.Debug($"repository url={repositoryUrl}"); executionContext.Debug($"targetPath={targetPath}"); executionContext.Debug($"sourceBranch={sourceBranch}"); @@ -355,6 +359,7 @@ public async Task GetSourceAsync( executionContext.Debug($"fetchTags={fetchTags}"); executionContext.Debug($"gitLfsSupport={gitLfsSupport}"); executionContext.Debug($"acceptUntrustedCerts={acceptUntrustedCerts}"); + executionContext.Debug($"sparseCheckout={enableSparseCheckout}"); bool schannelSslBackend = StringUtil.ConvertToBoolean(executionContext.Variables.GetValueOrDefault("agent.gituseschannel")?.Value); executionContext.Debug($"schannelSslBackend={schannelSslBackend}"); @@ -969,6 +974,30 @@ public async Task GetSourceAsync( } } + if (AgentKnobs.UseSparseCheckoutInCheckoutTask.GetValue(executionContext).AsBoolean()) + { + if (enableSparseCheckout) + { + // Set up sparse checkout + int exitCode_sparseCheckout = await gitCommandManager.GitSparseCheckout(executionContext, targetPath, sparseCheckoutDirectories, sparseCheckoutPatterns, cancellationToken); + + if (exitCode_sparseCheckout != 0) + { + throw new InvalidOperationException($"Git sparse checkout failed with exit code: {exitCode_sparseCheckout}"); + } + } + else + { + // Disable sparse checkout in case it was enabled in a previous checkout + int exitCode_sparseCheckoutDisable = await gitCommandManager.GitSparseCheckoutDisable(executionContext, targetPath, cancellationToken); + + if (exitCode_sparseCheckoutDisable != 0) + { + throw new InvalidOperationException($"Git sparse checkout disable failed with exit code: {exitCode_sparseCheckoutDisable}"); + } + } + } + // Finally, checkout the sourcesToBuild (if we didn't find a valid git object this will throw) int exitCode_checkout = await gitCommandManager.GitCheckout(executionContext, targetPath, sourcesToBuild, string.Join(" ", additionalCheckoutArgs), cancellationToken); if (exitCode_checkout != 0) diff --git a/src/Agent.Sdk/Knob/AgentKnobs.cs b/src/Agent.Sdk/Knob/AgentKnobs.cs index ad832c4d1d..25ff9db01d 100644 --- a/src/Agent.Sdk/Knob/AgentKnobs.cs +++ b/src/Agent.Sdk/Knob/AgentKnobs.cs @@ -773,5 +773,11 @@ public class AgentKnobs new EnvironmentKnobSource("AGENT_INSTALL_LEGACY_TF_EXE"), new PipelineFeatureSource("InstallLegacyTfExe"), new BuiltInDefaultKnobSource("false")); + + public static readonly Knob UseSparseCheckoutInCheckoutTask = new Knob( + nameof(UseSparseCheckoutInCheckoutTask), + "If true, agent will use sparse checkout in checkout task.", + new RuntimeKnobSource("AGENT_USE_SPARSE_CHECKOUT_IN_CHECKOUT_TASK"), + new BuiltInDefaultKnobSource("false")); } } diff --git a/src/Common.props b/src/Common.props index e490e55df7..2fa8e3c0d8 100644 --- a/src/Common.props +++ b/src/Common.props @@ -11,7 +11,7 @@ OS_UNKNOWN ARCH_UNKNOWN - 0.5.246-private + 0.5.247-private $(CodeAnalysis) false false