From 29399849fcc53416e76d4cab23aea1dc25df0a7a Mon Sep 17 00:00:00 2001 From: Ivan Duplenskikh Date: Tue, 17 Oct 2023 19:26:39 +0200 Subject: [PATCH 1/5] Add FF for setting git config --- src/Agent.Plugins/GitSourceProvider.cs | 11 +++++++++++ src/Agent.Sdk/Knob/AgentKnobs.cs | 7 +++++++ 2 files changed, 18 insertions(+) diff --git a/src/Agent.Plugins/GitSourceProvider.cs b/src/Agent.Plugins/GitSourceProvider.cs index 6af5fd4804..38a2750c6c 100644 --- a/src/Agent.Plugins/GitSourceProvider.cs +++ b/src/Agent.Plugins/GitSourceProvider.cs @@ -684,6 +684,17 @@ public async Task GetSourceAsync( executionContext.Warning("Unable turn off git auto garbage collection, git fetch operation may trigger auto garbage collection which will affect the performance of fetching."); } + if (AgentKnobs.FixPossibleGitOutOfMemoryProblem.GetValue(executionContext).AsBoolean()) + { + await gitCommandManager.GitConfig(executionContext, targetPath, "pack.threads", "1"); + await gitCommandManager.GitConfig(executionContext, targetPath, "http.postBuffer", "524288000"); + await gitCommandManager.GitConfig(executionContext, targetPath, "core.packedgitwindowsize", "256m"); + await gitCommandManager.GitConfig(executionContext, targetPath, "core.packedgitlimit", "256m"); + await gitCommandManager.GitConfig(executionContext, targetPath, "pack.windowmemory", "256m"); + await gitCommandManager.GitConfig(executionContext, targetPath, "pack.deltaCacheSize", "256m"); + await gitCommandManager.GitConfig(executionContext, targetPath, "pack.packSizeLimit", "256m"); + } + // always remove any possible left extraheader setting from git config. if (await gitCommandManager.GitConfigExist(executionContext, targetPath, $"http.{repositoryUrl.AbsoluteUri}.extraheader")) { diff --git a/src/Agent.Sdk/Knob/AgentKnobs.cs b/src/Agent.Sdk/Knob/AgentKnobs.cs index 36d1c7ed97..59c5a2829a 100644 --- a/src/Agent.Sdk/Knob/AgentKnobs.cs +++ b/src/Agent.Sdk/Knob/AgentKnobs.cs @@ -121,6 +121,13 @@ public class AgentKnobs new EnvironmentKnobSource("AGENT_GIT_USE_SECURE_PARAMETER_PASSING"), new BuiltInDefaultKnobSource("true")); + public static readonly Knob FixPossibleGitOutOfMemoryProblem = new Knob( + nameof(FixPossibleGitOutOfMemoryProblem), + "When true, set config git properties to fix possible out of memory problem", + new RuntimeKnobSource("FIX_POSSIBLE_GIT_OUT_OF_MEMORY_PROBLEM"), + new EnvironmentKnobSource("FIX_POSSIBLE_GIT_OUT_OF_MEMORY_PROBLEM"), + new BuiltInDefaultKnobSource("false")); + public static readonly Knob TfVCUseSecureParameterPassing = new Knob( nameof(TfVCUseSecureParameterPassing), "If true, don't pass auth token in TFVC parameters", From c6cc09e8648e8e341f6c2c8e43cca4123b64dddd Mon Sep 17 00:00:00 2001 From: Ivan Duplenskikh Date: Wed, 18 Oct 2023 23:50:08 +0200 Subject: [PATCH 2/5] Execute another git version when ff is enabled --- src/Agent.Plugins/GitCliManager.cs | 11 ++++++++++- src/Misc/externals.sh | 4 +++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/src/Agent.Plugins/GitCliManager.cs b/src/Agent.Plugins/GitCliManager.cs index ca36a5c69d..8e33819d5a 100644 --- a/src/Agent.Plugins/GitCliManager.cs +++ b/src/Agent.Plugins/GitCliManager.cs @@ -87,7 +87,16 @@ public virtual async Task LoadGitExecutionInfo(AgentTaskPluginExecutionContext c { string agentHomeDir = context.Variables.GetValueOrDefault("agent.homedirectory")?.Value; ArgUtil.NotNullOrEmpty(agentHomeDir, nameof(agentHomeDir)); - gitPath = Path.Combine(agentHomeDir, "externals", "git", "cmd", $"git.exe"); + + if (AgentKnobs.FixPossibleGitOutOfMemoryProblem.GetValue(context).AsBoolean()) + { + gitPath = Path.Combine(agentHomeDir, "externals", "git_ff", "cmd", $"git.exe"); + } + else + { + gitPath = Path.Combine(agentHomeDir, "externals", "git", "cmd", $"git.exe"); + } + gitLfsPath = Path.Combine(agentHomeDir, "externals", "git", PlatformUtil.BuiltOnX86 ? "mingw32" : "mingw64", "bin", "git-lfs.exe"); // Prepend the PATH. diff --git a/src/Misc/externals.sh b/src/Misc/externals.sh index 1b49a8cc78..52ee3f80cc 100644 --- a/src/Misc/externals.sh +++ b/src/Misc/externals.sh @@ -18,7 +18,8 @@ NODE_VERSION="6.17.1" NODE10_VERSION="10.24.1" NODE16_VERSION="16.20.2" NODE20_VERSION="20.3.1" -MINGIT_VERSION="2.39.1" +FF_MINGIT_VERSION="2.39.1" +MINGIT_VERSION="2.42.0" LFS_VERSION="3.3.0" get_abs_path() { @@ -160,6 +161,7 @@ if [[ "$PACKAGERUNTIME" == "win-x"* ]]; then acquireExternalTool "$CONTAINER_URL/vstsom/m122_887c6659/vstsom.zip" vstsom fi + acquireExternalTool "$CONTAINER_URL/mingit/${FF_MINGIT_VERSION}/MinGit-${FF_MINGIT_VERSION}-${BIT}-bit.zip" ff_git acquireExternalTool "$CONTAINER_URL/mingit/${MINGIT_VERSION}/MinGit-${MINGIT_VERSION}-${BIT}-bit.zip" git acquireExternalTool "$CONTAINER_URL/git-lfs/${LFS_VERSION}/x${BIT}/git-lfs.exe" "git/mingw${BIT}/bin" acquireExternalTool "$CONTAINER_URL/pdbstr/1/pdbstr.zip" pdbstr From 9c780f05119f0d663cae2676c42f99300a0ae0c9 Mon Sep 17 00:00:00 2001 From: Ivan Duplenskikh Date: Wed, 18 Oct 2023 23:51:20 +0200 Subject: [PATCH 3/5] Update path --- src/Agent.Plugins/GitCliManager.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Agent.Plugins/GitCliManager.cs b/src/Agent.Plugins/GitCliManager.cs index 8e33819d5a..ff26e269d1 100644 --- a/src/Agent.Plugins/GitCliManager.cs +++ b/src/Agent.Plugins/GitCliManager.cs @@ -90,7 +90,7 @@ public virtual async Task LoadGitExecutionInfo(AgentTaskPluginExecutionContext c if (AgentKnobs.FixPossibleGitOutOfMemoryProblem.GetValue(context).AsBoolean()) { - gitPath = Path.Combine(agentHomeDir, "externals", "git_ff", "cmd", $"git.exe"); + gitPath = Path.Combine(agentHomeDir, "externals", "ff_git", "cmd", $"git.exe"); } else { From b6b901e9d976d70be6aaeef63bdf626f61bd7395 Mon Sep 17 00:00:00 2001 From: Ivan Duplenskikh Date: Thu, 19 Oct 2023 10:21:18 +0200 Subject: [PATCH 4/5] Update MinGit version --- src/Misc/externals.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Misc/externals.sh b/src/Misc/externals.sh index 52ee3f80cc..2deb4d9db5 100644 --- a/src/Misc/externals.sh +++ b/src/Misc/externals.sh @@ -19,7 +19,7 @@ NODE10_VERSION="10.24.1" NODE16_VERSION="16.20.2" NODE20_VERSION="20.3.1" FF_MINGIT_VERSION="2.39.1" -MINGIT_VERSION="2.42.0" +MINGIT_VERSION="2.42.0.2" LFS_VERSION="3.3.0" get_abs_path() { From 4b624594ec0c275cc731dd821255c17280b497cf Mon Sep 17 00:00:00 2001 From: Ivan Duplenskikh Date: Thu, 19 Oct 2023 15:48:32 +0200 Subject: [PATCH 5/5] Set 2.42.0.2 version as ff and 2.39.1 as base --- src/Misc/externals.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Misc/externals.sh b/src/Misc/externals.sh index 2deb4d9db5..81dd68f16f 100644 --- a/src/Misc/externals.sh +++ b/src/Misc/externals.sh @@ -18,8 +18,8 @@ NODE_VERSION="6.17.1" NODE10_VERSION="10.24.1" NODE16_VERSION="16.20.2" NODE20_VERSION="20.3.1" -FF_MINGIT_VERSION="2.39.1" -MINGIT_VERSION="2.42.0.2" +MINGIT_VERSION="2.39.1" +FF_MINGIT_VERSION="2.42.0.2" LFS_VERSION="3.3.0" get_abs_path() {