Add networked builders

typescript/sdk/src/aws/s3.ts

@@ -0,0 +1,76 @@
+import { GetObjectCommand, S3Client } from '@aws-sdk/client-s3';
+import { Readable } from 'stream';
+
+import { streamToString } from '@hyperlane-xyz/utils';
+
+export const S3_BUCKET_REGEX =
+  /^(?:https?:\/\/)?(.*)\.s3\.(.*)\.amazonaws.com\/?$/;
+
+export interface S3Receipt<T = unknown> {
+  data: T;
+  modified: Date;
+}
+
+export class S3Wrapper {
+  private readonly client: S3Client;
+  readonly bucket: string;
+  readonly region: string;
+  readonly folder: string | undefined;
+
+  private cache: Record<string, S3Receipt<any>> | undefined;
+
+  static fromBucketUrl(bucketUrl: string): S3Wrapper {
+    const match = bucketUrl.match(S3_BUCKET_REGEX);
+    if (!match) throw new Error('Could not parse bucket url');
+    return new S3Wrapper(match[1], match[2], undefined);
+  }
+
+  constructor(
+    bucket: string,
+    region: string,
+    folder: string | undefined,
+    caching = false,
+  ) {
+    this.bucket = bucket;
+    this.region = region;
+    this.folder = folder;
+    this.client = new S3Client({ region });
+    if (caching) {
+      this.cache = {};
+    }
+  }
+
+  async getS3Obj<T>(key: string): Promise<S3Receipt<T> | undefined> {
+    const Key = this.folder ? `${this.folder}/${key}` : key;
+    if (this.cache?.[Key]) {
+      return this.cache![Key];
+    }
+
+    const command = new GetObjectCommand({
+      Bucket: this.bucket,
+      Key,
+    });
+    try {
+      const response = await this.client.send(command);
+      const body: string = await streamToString(response.Body as Readable);
+      const result = {
+        data: JSON.parse(body),
+        modified: response.LastModified!,
+      };
+      if (this.cache) {
+        this.cache[Key] = result;
+      }
+      return result;
+    } catch (e: any) {
+      if (e.message.includes('The specified key does not exist.')) {
+        return;
+      }
+      throw e;
+    }
+  }
+
+  url(key: string): string {
+    const Key = this.folder ? `${this.folder}/${key}` : key;
+    return `https://${this.bucket}.${this.region}.s3.amazonaws.com/${Key}`;
+  }
+}

typescript/sdk/src/aws/validator.ts

@@ -0,0 +1,103 @@
+import {
+  Announcement,
+  BaseValidator,
+  S3Announcement,
+  S3CheckpointWithId,
+  isS3CheckpointWithId,
+} from '@hyperlane-xyz/utils';
+
+import { S3Wrapper } from './s3.js';
+
+const checkpointWithMessageIdKey = (checkpointIndex: number) =>
+  `checkpoint_${checkpointIndex}_with_id.json`;
+const LATEST_KEY = 'checkpoint_latest_index.json';
+const ANNOUNCEMENT_KEY = 'announcement.json';
+const LOCATION_PREFIX = 's3://';
+
+/**
+ * Extension of BaseValidator that includes AWS S3 utilities.
+ */
+export class S3Validator extends BaseValidator {
+  public s3Bucket: S3Wrapper;
+
+  constructor(
+    address: string,
+    localDomain: number,
+    mailbox: string,
+    s3Bucket: string,
+    s3Region: string,
+    s3Folder: string | undefined,
+  ) {
+    super(address, localDomain, mailbox);
+    this.s3Bucket = new S3Wrapper(s3Bucket, s3Region, s3Folder, true); // caching enabled
+  }
+
+  static async fromStorageLocation(
+    storageLocation: string,
+  ): Promise<S3Validator> {
+    if (storageLocation.startsWith(LOCATION_PREFIX)) {
+      const suffix = storageLocation.slice(LOCATION_PREFIX.length);
+      const pieces = suffix.split('/');
+      if (pieces.length >= 2) {
+        const s3Bucket = new S3Wrapper(pieces[0], pieces[1], pieces[2]);
+        const announcement = await s3Bucket.getS3Obj<S3Announcement>(
+          ANNOUNCEMENT_KEY,
+        );
+        if (!announcement) {
+          throw new Error('No announcement found');
+        }
+
+        const address = announcement.data.value.validator;
+        const mailbox = announcement.data.value.mailbox_address;
+        const localDomain = announcement.data.value.mailbox_domain;
+
+        return new S3Validator(
+          address,
+          localDomain,
+          mailbox,
+          pieces[0],
+          pieces[1],
+          pieces[2],
+        );
+      }
+    }
+    throw new Error(`Unable to parse location ${storageLocation}`);
+  }
+
+  async getAnnouncement(): Promise<Announcement> {
+    const resp = await this.s3Bucket.getS3Obj<S3Announcement>(ANNOUNCEMENT_KEY);
+    if (!resp) {
+      throw new Error('No announcement found');
+    }
+
+    return resp.data.value;
+  }
+
+  async getCheckpoint(index: number) {
+    const key = checkpointWithMessageIdKey(index);
+    const s3Object = await this.s3Bucket.getS3Obj<S3CheckpointWithId>(key);
+    if (!s3Object) {
+      return;
+    }
+
+    if (isS3CheckpointWithId(s3Object.data)) {
+      return s3Object.data;
+    } else {
+      throw new Error('Failed to parse checkpoint');
+    }
+  }
+
+  async getLatestCheckpointIndex() {
+    const latestCheckpointIndex = await this.s3Bucket.getS3Obj<number>(
+      LATEST_KEY,
+    );
+
+    if (!latestCheckpointIndex) return -1;
+
+    return latestCheckpointIndex.data;
+  }
+
+  storageLocation(): string {
+    return `${LOCATION_PREFIX}/${this.s3Bucket.bucket}/${this.s3Bucket.region}`;
+  }
+}

typescript/sdk/src/hook/read.ts

@@ -8,7 +8,6 @@ import {
   FallbackDomainRoutingHook__factory,
   IPostDispatchHook__factory,
   InterchainGasPaymaster__factory,
-  MerkleTreeHook__factory,
   OPStackHook__factory,
   PausableHook__factory,
   ProtocolFee__factory,
@@ -42,6 +41,8 @@ import {
   RoutingHookConfig,
 } from './types.js';
 
+export type DerivedHookConfigWithAddress = WithAddress<HookConfig>;
+
 export interface HookReader {
   deriveHookConfig(address: Address): Promise<WithAddress<HookConfig>>;
   deriveMerkleTreeConfig(
@@ -82,9 +83,12 @@ export class EvmHookReader implements HookReader {
     this.provider = multiProvider.getProvider(chain);
   }
 
-  async deriveHookConfig(address: Address): Promise<WithAddress<HookConfig>> {
+  async deriveHookConfig(
+    address: Address,
+  ): Promise<DerivedHookConfigWithAddress> {
     const hook = IPostDispatchHook__factory.connect(address, this.provider);
     const onchainHookType: OnchainHookType = await hook.hookType();
+    this.logger.debug('Deriving HookConfig', { address, onchainHookType });
 
     switch (onchainHookType) {
       case OnchainHookType.ROUTING:
@@ -115,8 +119,8 @@ export class EvmHookReader implements HookReader {
   async deriveMerkleTreeConfig(
     address: Address,
   ): Promise<WithAddress<MerkleTreeHookConfig>> {
-    const hook = MerkleTreeHook__factory.connect(address, this.provider);
-    assert((await hook.hookType()) === OnchainHookType.MERKLE_TREE);
+    // const hook = MerkleTreeHook__factory.connect(address, this.provider);
+    // assert((await hook.hookType()) === OnchainHookType.MERKLE_TREE);
 
     return {
       address,
@@ -128,7 +132,7 @@ export class EvmHookReader implements HookReader {
     address: Address,
   ): Promise<WithAddress<AggregationHookConfig>> {
     const hook = StaticAggregationHook__factory.connect(address, this.provider);
-    assert((await hook.hookType()) === OnchainHookType.AGGREGATION);
+    // assert((await hook.hookType()) === OnchainHookType.AGGREGATION);
 
     const hooks = await hook.hooks(ethers.constants.AddressZero);
     const hookConfigs = await concurrentMap(
@@ -149,9 +153,9 @@ export class EvmHookReader implements HookReader {
       address,
       this.provider,
     );
-    assert(
-      (await hook.hookType()) === OnchainHookType.INTERCHAIN_GAS_PAYMASTER,
-    );
+    // assert(
+    //   (await hook.hookType()) === OnchainHookType.INTERCHAIN_GAS_PAYMASTER,
+    // );
 
     const owner = await hook.owner();
     const beneficiary = await hook.beneficiary();
@@ -220,7 +224,7 @@ export class EvmHookReader implements HookReader {
     address: Address,
   ): Promise<WithAddress<ProtocolFeeHookConfig>> {
     const hook = ProtocolFee__factory.connect(address, this.provider);
-    assert((await hook.hookType()) === OnchainHookType.PROTOCOL_FEE);
+    // assert((await hook.hookType()) === OnchainHookType.PROTOCOL_FEE);
 
     const owner = await hook.owner();
     const maxProtocolFee = await hook.MAX_PROTOCOL_FEE();
@@ -242,7 +246,7 @@ export class EvmHookReader implements HookReader {
   ): Promise<WithAddress<OpStackHookConfig>> {
     const hook = OPStackHook__factory.connect(address, this.provider);
     const owner = await hook.owner();
-    assert((await hook.hookType()) === OnchainHookType.ID_AUTH_ISM);
+    // assert((await hook.hookType()) === OnchainHookType.ID_AUTH_ISM);
 
     const messengerContract = await hook.l1Messenger();
     const destinationDomain = await hook.destinationDomain();
@@ -262,7 +266,7 @@ export class EvmHookReader implements HookReader {
     address: Address,
   ): Promise<WithAddress<DomainRoutingHookConfig>> {
     const hook = DomainRoutingHook__factory.connect(address, this.provider);
-    assert((await hook.hookType()) === OnchainHookType.ROUTING);
+    // assert((await hook.hookType()) === OnchainHookType.ROUTING);
 
     const owner = await hook.owner();
     const domainHooks = await this.fetchDomainHooks(hook);
@@ -282,7 +286,7 @@ export class EvmHookReader implements HookReader {
       address,
       this.provider,
     );
-    assert((await hook.hookType()) === OnchainHookType.FALLBACK_ROUTING);
+    // assert((await hook.hookType()) === OnchainHookType.FALLBACK_ROUTING);
 
     const owner = await hook.owner();
     const domainHooks = await this.fetchDomainHooks(hook);
@@ -328,7 +332,7 @@ export class EvmHookReader implements HookReader {
     address: Address,
   ): Promise<WithAddress<PausableHookConfig>> {
     const hook = PausableHook__factory.connect(address, this.provider);
-    assert((await hook.hookType()) === OnchainHookType.PAUSABLE);
+    // assert((await hook.hookType()) === OnchainHookType.PAUSABLE);
 
     const owner = await hook.owner();
     return {

typescript/sdk/src/ism/metadata/aggregation.ts

@@ -1,4 +1,19 @@
-import { fromHexString, toHexString } from '@hyperlane-xyz/utils';
+import { TransactionReceipt } from '@ethersproject/providers';
+
+import {
+  WithAddress,
+  assert,
+  fromHexString,
+  rootLogger,
+  toHexString,
+} from '@hyperlane-xyz/utils';
+
+import { DispatchedMessage } from '../../core/types.js';
+import { DerivedHookConfigWithAddress } from '../../hook/read.js';
+import { DerivedIsmConfigWithAddress } from '../read.js';
+import { AggregationIsmConfig } from '../types.js';
+
+import { BaseMetadataBuilder, MetadataBuilder } from './builder.js';
 
 // null indicates that metadata is NOT INCLUDED for this submodule
 // empty or 0x string indicates that metadata is INCLUDED but NULL
@@ -9,7 +24,54 @@ export interface AggregationIsmMetadata {
 const RANGE_SIZE = 4;
 
 // adapted from rust/agents/relayer/src/msg/metadata/aggregation.rs
-export class AggregationIsmMetadataBuilder {
+export class AggregationIsmMetadataBuilder
+  implements
+    MetadataBuilder<
+      WithAddress<AggregationIsmConfig>,
+      DerivedHookConfigWithAddress
+    >
+{
+  protected logger = rootLogger.child({
+    module: 'AggregationIsmMetadataBuilder',
+  });
+
+  constructor(protected readonly base: BaseMetadataBuilder) {}
+
+  async build(
+    message: DispatchedMessage,
+    context: {
+      ism: WithAddress<AggregationIsmConfig>;
+      hook: DerivedHookConfigWithAddress;
+      dispatchTx: TransactionReceipt;
+    },
+    maxDepth = 10,
+  ): Promise<string> {
+    assert(maxDepth > 0, 'Max depth reached');
+    const promises = await Promise.allSettled(
+      context.ism.modules.map((module) =>
+        this.base.build(
+          message,
+          {
+            ...context,
+            ism: module as DerivedIsmConfigWithAddress,
+          },
+          maxDepth - 1,
+        ),
+      ),
+    );
+    const submoduleMetadata = promises.map((r) =>
+      r.status === 'fulfilled' ? r.value : null,
+    );
+    const included = submoduleMetadata.filter((m) => m !== null).length;
+    if (included < context.ism.threshold) {
+      throw new Error(
+        `Only built ${included} of ${context.ism.threshold} required modules`,
+      );
+    }
+
+    return AggregationIsmMetadataBuilder.encode({ submoduleMetadata });
+  }
+
   static rangeIndex(index: number): number {
     return index * 2 * RANGE_SIZE;
   }