Add PartialRegistry class and merge method

.changeset/brown-taxis-invite.md

@@ -0,0 +1,6 @@
+---
+'@hyperlane-xyz/registry': minor
+---
+
+Add PartialRegistry class
+Add merge() method to IRegistry

.changeset/thick-weeks-approve.md

@@ -0,0 +1,5 @@
+---
+'@hyperlane-xyz/registry': minor
+---
+
+Add MergedRegistry class

src/index.ts

@@ -4,4 +4,6 @@ export { DEFAULT_GITHUB_REGISTRY } from './consts.js';
 export { BaseRegistry, CHAIN_FILE_REGEX } from './registry/BaseRegistry.js';
 export { GithubRegistry, GithubRegistryOptions } from './registry/GithubRegistry.js';
 export { ChainFiles, IRegistry, RegistryContent, RegistryType } from './registry/IRegistry.js';
+export { MergedRegistry, MergedRegistryOptions } from './registry/MergedRegistry.js';
+export { PartialRegistry, PartialRegistryOptions } from './registry/PartialRegistry.js';
 export { ChainAddresses, ChainAddressesSchema } from './types.js';

src/registry/BaseRegistry.ts

@@ -3,6 +3,7 @@ import type { Logger } from 'pino';
 import type { ChainMap, ChainMetadata, ChainName, WarpCoreConfig } from '@hyperlane-xyz/sdk';
 import type { ChainAddresses, MaybePromise } from '../types.js';
 import type { IRegistry, RegistryContent, RegistryType } from './IRegistry.js';
+import { MergedRegistry } from './MergedRegistry.js';
 
 export const CHAIN_FILE_REGEX = /chains\/([a-z0-9]+)\/([a-z]+)\.(yaml|svg)/;
 
@@ -72,4 +73,8 @@ export abstract class BaseRegistry implements IRegistry {
   }): MaybePromise<void>;
   abstract removeChain(chain: ChainName): MaybePromise<void>;
   abstract addWarpRoute(config: WarpCoreConfig): MaybePromise<void>;
+
+  merge(otherRegistry: IRegistry): IRegistry {
+    return new MergedRegistry({ registries: [this, otherRegistry], logger: this.logger });
+  }
 }

src/registry/GithubRegistry.ts

@@ -29,6 +29,11 @@ interface TreeNode {
   url: string;
 }
 
+/**
+ * A registry that uses a github repository as its data source.
+ * Reads are performed via the github API and github's raw content URLs.
+ * Writes are not yet supported (TODO)
+ */
 export class GithubRegistry extends BaseRegistry implements IRegistry {
   public readonly type = RegistryType.Github;
   public readonly url: URL;

src/registry/IRegistry.ts

@@ -17,6 +17,8 @@ export interface RegistryContent {
 export enum RegistryType {
   Github = 'github',
   Local = 'local',
+  Merged = 'merged',
+  Partial = 'partial',
 }
 
 export interface IRegistry {
@@ -49,4 +51,6 @@ export interface IRegistry {
 
   addWarpRoute(config: WarpCoreConfig): MaybePromise<void>;
   // TODO define more deployment artifact related methods
+
+  merge(otherRegistry: IRegistry): IRegistry;
 }

src/registry/LocalRegistry.ts

@@ -8,21 +8,26 @@ import type { ChainMap, ChainMetadata, ChainName, WarpCoreConfig } from '@hyperl
 import { SCHEMA_REF } from '../consts.js';
 import { ChainAddresses, ChainAddressesSchema } from '../types.js';
 import { toYamlString } from '../utils.js';
-import { BaseRegistry, CHAIN_FILE_REGEX } from './BaseRegistry.js';
+import { CHAIN_FILE_REGEX } from './BaseRegistry.js';
 import {
   RegistryType,
   type ChainFiles,
   type IRegistry,
   type RegistryContent,
 } from './IRegistry.js';
+import { SynchronousRegistry } from './SynchronousRegistry.js';
 import { warpConfigToWarpAddresses } from './warp-utils.js';
 
 export interface LocalRegistryOptions {
   uri: string;
   logger?: Logger;
 }
 
-export class LocalRegistry extends BaseRegistry implements IRegistry {
+/**
+ * A registry that uses a local file system path as its data source.
+ * Requires file system access so it cannot be used in the browser.
+ */
+export class LocalRegistry extends SynchronousRegistry implements IRegistry {
   public readonly type = RegistryType.Local;
 
   constructor(options: LocalRegistryOptions) {
@@ -48,10 +53,6 @@ export class LocalRegistry extends BaseRegistry implements IRegistry {
     return (this.listContentCache = { chains, deployments: {} });
   }
 
-  getChains(): Array<ChainName> {
-    return Object.keys(this.listRegistryContent().chains);
-  }
-
   getMetadata(): ChainMap<ChainMetadata> {
     if (this.metadataCache) return this.metadataCache;
     const chainMetadata: ChainMap<ChainMetadata> = {};
@@ -64,11 +65,6 @@ export class LocalRegistry extends BaseRegistry implements IRegistry {
     return (this.metadataCache = chainMetadata);
   }
 
-  getChainMetadata(chainName: ChainName): ChainMetadata | null {
-    const metadata = this.getMetadata();
-    return metadata[chainName] ?? null;
-  }
-
   getAddresses(): ChainMap<ChainAddresses> {
     if (this.addressCache) return this.addressCache;
     const chainAddresses: ChainMap<ChainAddresses> = {};
@@ -81,43 +77,10 @@ export class LocalRegistry extends BaseRegistry implements IRegistry {
     return (this.addressCache = chainAddresses);
   }
 
-  getChainAddresses(chainName: ChainName): ChainAddresses | null {
-    const addresses = this.getAddresses();
-    return addresses[chainName] ?? null;
-  }
-
-  addChain(chain: {
-    chainName: ChainName;
-    metadata?: ChainMetadata;
-    addresses?: ChainAddresses;
-  }): void {
-    const currentChains = this.listRegistryContent().chains;
-    if (currentChains[chain.chainName])
-      throw new Error(`Chain ${chain.chainName} already exists in registry`);
-
-    this.createOrUpdateChain(chain);
-  }
-
-  updateChain(chain: {
-    chainName: ChainName;
-    metadata?: ChainMetadata;
-    addresses?: ChainAddresses;
-  }): void {
-    const currentChains = this.listRegistryContent();
-    if (!currentChains.chains[chain.chainName]) {
-      this.logger.debug(`Chain ${chain.chainName} not found in registry, adding it now`);
-    }
-    this.createOrUpdateChain(chain);
-  }
-
   removeChain(chainName: ChainName): void {
-    const currentChains = this.listRegistryContent().chains;
-    if (!currentChains[chainName]) throw new Error(`Chain ${chainName} does not exist in registry`);
-
-    this.removeFiles(Object.values(currentChains[chainName]));
-    if (this.listContentCache?.chains[chainName]) delete this.listContentCache.chains[chainName];
-    if (this.metadataCache?.[chainName]) delete this.metadataCache[chainName];
-    if (this.addressCache?.[chainName]) delete this.addressCache[chainName];
+    const chainFiles = this.listRegistryContent().chains[chainName];
+    super.removeChain(chainName);
+    this.removeFiles(Object.values(chainFiles));
   }
 
   protected listFiles(dirPath: string): string[] {

src/registry/MergedRegistry.ts

@@ -0,0 +1,134 @@
+import type { Logger } from 'pino';
+
+import type { ChainMap, ChainMetadata, ChainName, WarpCoreConfig } from '@hyperlane-xyz/sdk';
+import { ChainAddresses } from '../types.js';
+import { objMerge } from '../utils.js';
+import { IRegistry, RegistryContent, RegistryType } from './IRegistry.js';
+
+export interface MergedRegistryOptions {
+  registries: Array<IRegistry>;
+  logger?: Logger;
+}
+
+/**
+ * A registry that accepts multiple sub-registries.
+ * Read methods are performed on all sub-registries and the results are merged.
+ * Write methods are performed on all sub-registries.
+ * Can be created manually or by calling `.merge()` on an existing registry.
+ */
+export class MergedRegistry implements IRegistry {
+  public readonly type = RegistryType.Merged;
+  public readonly uri = '__merged_registry__';
+  public readonly registries: Array<IRegistry>;
+  protected readonly logger: Logger;
+
+  constructor({ registries, logger }: MergedRegistryOptions) {
+    if (!registries.length) throw new Error('At least one registry URI is required');
+    this.registries = registries;
+    // @ts-ignore
+    this.logger = logger || console;
+  }
+
+  async listRegistryContent(): Promise<RegistryContent> {
+    const results = await this.multiRegistryRead((r) => r.listRegistryContent());
+    return results.reduce((acc, content) => objMerge(acc, content), {
+      chains: {},
+      deployments: {},
+    });
+  }
+
+  async getChains(): Promise<Array<ChainName>> {
+    return Object.keys(await this.getMetadata());
+  }
+
+  async getMetadata(): Promise<ChainMap<ChainMetadata>> {
+    const results = await this.multiRegistryRead((r) => r.getMetadata());
+    return results.reduce((acc, content) => objMerge(acc, content), {});
+  }
+
+  async getChainMetadata(chainName: ChainName): Promise<ChainMetadata | null> {
+    return (await this.getMetadata())[chainName] || null;
+  }
+
+  async getAddresses(): Promise<ChainMap<ChainAddresses>> {
+    const results = await this.multiRegistryRead((r) => r.getAddresses());
+    return results.reduce((acc, content) => objMerge(acc, content), {});
+  }
+
+  async getChainAddresses(chainName: ChainName): Promise<ChainAddresses | null> {
+    return (await this.getAddresses())[chainName] || null;
+  }
+
+  async getChainLogoUri(chainName: ChainName): Promise<string | null> {
+    const results = await this.multiRegistryRead((r) => r.getChainLogoUri(chainName));
+    return results.filter((uri) => uri !== null)[0] || null;
+  }
+
+  async addChain(chain: {
+    chainName: ChainName;
+    metadata?: ChainMetadata;
+    addresses?: ChainAddresses;
+  }): Promise<void> {
+    return this.multiRegistryWrite(
+      async (registry) => await registry.addChain(chain),
+      `adding chain ${chain.chainName}`,
+    );
+  }
+
+  async updateChain(chain: {
+    chainName: ChainName;
+    metadata?: ChainMetadata;
+    addresses?: ChainAddresses;
+  }): Promise<void> {
+    return this.multiRegistryWrite(
+      async (registry) => await registry.updateChain(chain),
+      `updating chain ${chain.chainName}`,
+    );
+  }
+
+  async removeChain(chain: ChainName): Promise<void> {
+    return this.multiRegistryWrite(
+      async (registry) => await registry.removeChain(chain),
+      `removing chain ${chain}`,
+    );
+  }
+
+  async addWarpRoute(config: WarpCoreConfig): Promise<void> {
+    return this.multiRegistryWrite(
+      async (registry) => await registry.addWarpRoute(config),
+      'adding warp route',
+    );
+  }
+
+  protected async multiRegistryRead<R>(readFn: (registry: IRegistry) => Promise<R> | R) {
+    return Promise.all(this.registries.map(readFn));
+  }
+
+  protected async multiRegistryWrite(
+    writeFn: (registry: IRegistry) => Promise<void>,
+    logMsg: string,
+  ): Promise<void> {
+    for (const registry of this.registries) {
+      // TODO remove this when GithubRegistry supports write methods
+      if (registry.type === RegistryType.Github) {
+        this.logger.warn(`Skipping ${logMsg} at ${registry.type} registry`);
+        continue;
+      }
+      try {
+        this.logger.info(`Now ${logMsg} at ${registry.type} registry at ${registry.uri}`);
+        await writeFn(registry);
+        this.logger.info(`Done ${logMsg} at ${registry.type} registry`);
+      } catch (error) {
+        // To prevent loss of artifacts, MergedRegistry write methods are failure tolerant
+        this.logger.error(`Failure ${logMsg} at ${registry.type} registry`, error);
+      }
+    }
+  }
+
+  merge(otherRegistry: IRegistry): IRegistry {
+    return new MergedRegistry({
+      registries: [...this.registries, otherRegistry],
+      logger: this.logger,
+    });
+  }
+}

src/registry/PartialRegistry.ts

@@ -0,0 +1,74 @@
+import type { Logger } from 'pino';
+
+import type { ChainMap, ChainMetadata, ChainName, WarpCoreConfig } from '@hyperlane-xyz/sdk';
+import { ChainAddresses } from '../types.js';
+import { ChainFiles, IRegistry, RegistryContent, RegistryType } from './IRegistry.js';
+import { SynchronousRegistry } from './SynchronousRegistry.js';
+
+const PARTIAL_URI_PLACEHOLDER = '__partial_registry__';
+
+/**
+ * A registry that accepts partial data, such as incomplete chain metadata or addresses.
+ * Useful for merging with other registries force overrides of subsets of data.
+ */
+export interface PartialRegistryOptions {
+  chainMetadata?: ChainMap<Partial<ChainMetadata>>;
+  chainAddresses?: ChainMap<Partial<ChainAddresses>>;
+  // TODO add more fields here as needed
+  logger?: Logger;
+}
+
+export class PartialRegistry extends SynchronousRegistry implements IRegistry {
+  public readonly type = RegistryType.Partial;
+  public chainMetadata: ChainMap<Partial<ChainMetadata>>;
+  public chainAddresses: ChainMap<Partial<ChainAddresses>>;
+
+  constructor({ chainMetadata, chainAddresses, logger }: PartialRegistryOptions) {
+    super({ uri: PARTIAL_URI_PLACEHOLDER, logger });
+    this.chainMetadata = chainMetadata || {};
+    this.chainAddresses = chainAddresses || {};
+  }
+
+  listRegistryContent(): RegistryContent {
+    const chains: ChainMap<ChainFiles> = {};
+    Object.keys(this.chainMetadata).forEach((c) => {
+      chains[c] ||= {};
+      chains[c].metadata = PARTIAL_URI_PLACEHOLDER;
+    });
+    Object.keys(this.chainAddresses).forEach((c) => {
+      chains[c] ||= {};
+      chains[c].addresses = PARTIAL_URI_PLACEHOLDER;
+    });
+    return {
+      chains,
+      deployments: {},
+    };
+  }
+
+  getMetadata(): ChainMap<ChainMetadata> {
+    return this.chainMetadata as ChainMap<ChainMetadata>;
+  }
+
+  getAddresses(): ChainMap<ChainAddresses> {
+    return this.chainAddresses as ChainMap<ChainAddresses>;
+  }
+
+  removeChain(chainName: ChainName): void {
+    super.removeChain(chainName);
+    if (this.chainMetadata?.[chainName]) delete this.chainMetadata[chainName];
+    if (this.chainAddresses?.[chainName]) delete this.chainAddresses[chainName];
+  }
+
+  addWarpRoute(_config: WarpCoreConfig): void {
+    throw new Error('Method not implemented.');
+  }
+
+  protected createOrUpdateChain(chain: {
+    chainName: ChainName;
+    metadata?: ChainMetadata;
+    addresses?: ChainAddresses;
+  }): void {
+    if (chain.metadata) this.chainMetadata[chain.chainName] = chain.metadata;
+    if (chain.addresses) this.chainAddresses[chain.chainName] = chain.addresses;
+  }
+}