feat: bump relayer memory usage to 20GB, start using new SVM submissi…

…on in relayer (#5063)

### Description

- deploys #4959
- Sets a small priority fee for Eclipse following some recent spikes
there,
https://discord.com/channels/935678348330434570/1320851852157255701 for
context
- Makes the relayer request 20GB of memory, forcing relayers to have
their own node in our bigger node pool (capacity 29 GB)
- Driveby to use G (Gigabytes) instead of Gi (Gibibytes) to match what
GCP uses in their UI, also a bit more intuitive
- Added a `--concurrency <num>` flag that can be used when deploying
agents. Especially useful when releasing to all validators, which we
have tons of now.
- Concurrently refetching remote repositories on helm poses issues, so I
made this not happen by default anymore. It's not configurable at the
CLI level to enable it, but we have the ability to easily add it later.
Because our agent deploy doesn't actually require any remote helm repos,
it's not necessary
- already rolled out

### Drive-by changes

<!--
Are there any minor or drive-by changes also included?
-->

### Related issues

<!--
- Fixes #[issue number here]
-->

### Backward compatibility

<!--
Are these changes backward compatible? Are there any infrastructure
implications, e.g. changes that would prohibit deploying older commits
using this infra tooling?

Yes/No
-->

### Testing

<!--
What kind of testing have these changes undergone?

None/Manual/Unit Tests
-->

.changeset/stupid-seahorses-yell.md

@@ -0,0 +1,5 @@
+---
+'@hyperlane-xyz/utils': patch
+---
+
+Require concurrency > 0 for concurrentMap

typescript/infra/config/environments/mainnet3/agent.ts

@@ -414,6 +414,19 @@ const sealevelPriorityFeeOracleConfigGetter = (
       // URL is auto populated by the external secrets in the helm chart
       url: '',
     };
+  } else if (chain === 'eclipsemainnet') {
+    // As of Dec 23:
+    // Eclipse has recently seen some increased usage with their referral program,
+    // and we have had intermittent issues landing txs. Not many txs on Eclipse use
+    // priority fees, so we use a low priority fee.
+    return {
+      type: AgentSealevelPriorityFeeOracleType.Constant,
+      // 2000 micro lamports of ETH, which at a compute unit limit of 400K
+      // and an ETH price of $3450 (Dec 23, 2024) comes to about $0.00276 USD:
+      // >>> (((2000 / 1e6) * 400000) / 1e9) * 3450
+      // 0.00276
+      fee: '2000',
+    };
   }
 
   // For all other chains, we use the constant fee oracle with a fee of 0
@@ -530,21 +543,21 @@ const metricAppContextsGetter = (): MetricAppContext[] => {
 const relayerResources = {
   requests: {
     cpu: '14000m',
-    memory: '15Gi',
+    memory: '20G',
   },
 };
 
 const validatorResources = {
   requests: {
     cpu: '500m',
-    memory: '1Gi',
+    memory: '1G',
   },
 };
 
 const scraperResources = {
   requests: {
     cpu: '2000m',
-    memory: '4Gi',
+    memory: '4G',
   },
 };
 
@@ -605,7 +618,7 @@ const hyperlane: RootAgentConfig = {
     rpcConsensusType: RpcConsensusType.Fallback,
     docker: {
       repo,
-      tag: '7c0c967-20241218-173053',
+      tag: 'e9911bb-20241223-211526',
     },
     blacklist,
     gasPaymentEnforcement: gasPaymentEnforcement,
@@ -640,7 +653,7 @@ const releaseCandidate: RootAgentConfig = {
     rpcConsensusType: RpcConsensusType.Fallback,
     docker: {
       repo,
-      tag: '7c0c967-20241218-173053',
+      tag: 'e9911bb-20241223-211526',
     },
     blacklist,
     // We're temporarily (ab)using the RC relayer as a way to increase

typescript/infra/scripts/agent-utils.ts

@@ -261,6 +261,13 @@ export function withConcurrentDeploy<T>(args: Argv<T>) {
     .default('concurrentDeploy', false);
 }
 
+export function withConcurrency<T>(args: Argv<T>) {
+  return args
+    .describe('concurrency', 'Number of concurrent deploys')
+    .number('concurrency')
+    .default('concurrency', 1);
+}
+
 export function withRpcUrls<T>(args: Argv<T>) {
   return args
     .describe(

typescript/infra/scripts/agents/utils.ts

@@ -1,3 +1,5 @@
+import { concurrentMap } from '@hyperlane-xyz/utils';
+
 import {
   AgentHelmManager,
   RelayerHelmManager,
@@ -7,12 +9,13 @@ import {
 import { RootAgentConfig } from '../../src/config/agent/agent.js';
 import { EnvironmentConfig } from '../../src/config/environment.js';
 import { Role } from '../../src/roles.js';
-import { HelmCommand } from '../../src/utils/helm.js';
+import { HelmCommand, HelmManager } from '../../src/utils/helm.js';
 import {
   assertCorrectKubeContext,
   getArgs,
   withAgentRolesRequired,
   withChains,
+  withConcurrency,
   withContext,
 } from '../agent-utils.js';
 import { getConfigsBasedOnArgs } from '../core-utils.js';
@@ -24,6 +27,7 @@ export class AgentCli {
   initialized = false;
   dryRun = false;
   chains?: string[];
+  concurrency = 1;
 
   public async runHelmCommand(command: HelmCommand) {
     await this.init();
@@ -63,15 +67,20 @@ export class AgentCli {
       console.log('Dry run values:\n', JSON.stringify(values, null, 2));
     }
 
-    for (const m of Object.values(managers)) {
-      await m.runHelmCommand(command, this.dryRun);
-    }
+    await concurrentMap(
+      this.concurrency,
+      Object.entries(managers),
+      async ([key, manager]) => {
+        console.log(`Running helm command for ${key}`);
+        await manager.runHelmCommand(command, { dryRun: this.dryRun });
+      },
+    );
   }
 
   protected async init() {
     if (this.initialized) return;
-    const argv = await withChains(
-      withAgentRolesRequired(withContext(getArgs())),
+    const argv = await withConcurrency(
+      withChains(withAgentRolesRequired(withContext(getArgs()))),
     )
       .describe('dry-run', 'Run through the steps without making any changes')
       .boolean('dry-run').argv;
@@ -92,5 +101,6 @@ export class AgentCli {
     this.dryRun = argv.dryRun || false;
     this.initialized = true;
     this.chains = argv.chains;
+    this.concurrency = argv.concurrency;
   }
 }

typescript/infra/src/helloworld/kathy.ts

@@ -15,6 +15,7 @@ import {
 import { Role } from '../roles.js';
 import {
   HelmCommand,
+  HelmCommandOptions,
   HelmManager,
   HelmValues,
   helmifyValues,
@@ -90,7 +91,10 @@ export class KathyHelmManager extends HelmManager<HelmValues> {
     };
   }
 
-  async runHelmCommand(action: HelmCommand, dryRun?: boolean): Promise<void> {
+  async runHelmCommand(
+    action: HelmCommand,
+    options?: HelmCommandOptions,
+  ): Promise<void> {
     // If using AWS keys, ensure the Kathy user and key has been created
     if (this.agentConfig.aws) {
       const awsUser = new AgentAwsUser(
@@ -112,6 +116,6 @@ export class KathyHelmManager extends HelmManager<HelmValues> {
     );
     await kathyKey.createIfNotExists();
 
-    super.runHelmCommand(action, dryRun);
+    await super.runHelmCommand(action, options);
   }
 }

typescript/infra/src/utils/helm.ts

@@ -90,8 +90,17 @@ export function getDeployableHelmChartName(helmChartConfig: HelmChartConfig) {
   return helmChartConfig.name;
 }
 
-export function buildHelmChartDependencies(chartPath: string) {
-  return execCmd(`cd ${chartPath} && helm dependency build`, {}, false, true);
+export function buildHelmChartDependencies(
+  chartPath: string,
+  updateRepoCache: boolean,
+) {
+  const flags = updateRepoCache ? '' : '--skip-refresh';
+  return execCmd(
+    `cd ${chartPath} && helm dependency build ${flags}`,
+    {},
+    false,
+    true,
+  );
 }
 
 // Convenience function to remove a helm release without having a HelmManger for it.
@@ -101,6 +110,11 @@ export function removeHelmRelease(releaseName: string, namespace: string) {
 
 export type HelmValues = Record<string, any>;
 
+export interface HelmCommandOptions {
+  dryRun?: boolean;
+  updateRepoCache?: boolean;
+}
+
 export abstract class HelmManager<T = HelmValues> {
   abstract readonly helmReleaseName: string;
   abstract readonly helmChartPath: string;
@@ -112,7 +126,13 @@ export abstract class HelmManager<T = HelmValues> {
    */
   abstract helmValues(): Promise<T>;
 
-  async runHelmCommand(action: HelmCommand, dryRun?: boolean): Promise<void> {
+  async runHelmCommand(
+    action: HelmCommand,
+    options?: HelmCommandOptions,
+  ): Promise<void> {
+    const dryRun = options?.dryRun ?? false;
+    const updateRepoCache = options?.updateRepoCache ?? false;
+
     const cmd = ['helm', action];
     if (dryRun) cmd.push('--dry-run');
 
@@ -142,7 +162,7 @@ export abstract class HelmManager<T = HelmValues> {
       }
     }
 
-    await buildHelmChartDependencies(this.helmChartPath);
+    await buildHelmChartDependencies(this.helmChartPath, updateRepoCache);
     cmd.push(
       this.helmReleaseName,
       this.helmChartPath,

typescript/utils/src/async.ts

@@ -1,4 +1,5 @@
 import { rootLogger } from './logging.js';
+import { assert } from './validation.js';
 
 /**
  * Return a promise that resolves in ms milliseconds.
@@ -159,6 +160,7 @@ export async function concurrentMap<A, B>(
   mapFn: (val: A, idx: number) => Promise<B>,
 ): Promise<B[]> {
   let res: B[] = [];
+  assert(concurrency > 0, 'concurrency must be greater than 0');
   for (let i = 0; i < xs.length; i += concurrency) {
     const remaining = xs.length - i;
     const sliceSize = Math.min(remaining, concurrency);