From c2abdeb97157ae2cfc43c8c69c0d472c0ba0e1a6 Mon Sep 17 00:00:00 2001
From: Daniel Beal <git@dbeal.dev>
Date: Tue, 30 Jul 2024 17:41:31 +0900
Subject: [PATCH 1/6] start working on auto verify

---
 packages/indexer/src/auto-verify.ts | 25 +++++++++++++++++++++++++
 packages/indexer/src/db.ts          |  1 +
 packages/indexer/src/registry.ts    |  2 ++
 3 files changed, 28 insertions(+)
 create mode 100644 packages/indexer/src/auto-verify.ts

diff --git a/packages/indexer/src/auto-verify.ts b/packages/indexer/src/auto-verify.ts
new file mode 100644
index 000000000..2bf8f1353
--- /dev/null
+++ b/packages/indexer/src/auto-verify.ts
@@ -0,0 +1,25 @@
+import _ from 'lodash';
+import * as rkey from './db';
+import { ActualRedisClientType, useRedis } from './redis';
+import { verify } from '@usecannon/cli/dist/src/commands/verify';
+import { resolveCliSettings } from '@usecannon/cli/dist/src/settings';
+
+/* eslint no-console: "off" */
+
+export async function loop() {
+  const cliSettings = resolveCliSettings();
+  const rdb = await useRedis();
+
+  const specialChainSettings = await import (process.cwd() + '/verify-chains.json');
+
+  let lastKey = rdb.get(rkey.RKEY_REGISTRY_STREAM + ':auto-verify-last');
+
+  const e = await rdb.xRead(rkey.RKEY_REGISTRY_STREAM, { BLOCK: true, COUNT: 1 });
+  while (e) {
+    // run the cannon verify command from the cli
+    await verify(e.packageUrl, _.defaults({ specialChainSettings[chainId.toString()] }, cliSettings), null, e.chainId);
+
+    let lastKey = rdb.set(rkey.RKEY_REGISTRY_STREAM + ':auto-verify-last', ++lastKey);
+    e = await rdb.xRead(rkey.RKEY_REGISTRY_STREAM, { BLOCK: true, COUNT: 1 });
+  }
+}
diff --git a/packages/indexer/src/db.ts b/packages/indexer/src/db.ts
index 0d6219489..6cb724b0e 100644
--- a/packages/indexer/src/db.ts
+++ b/packages/indexer/src/db.ts
@@ -24,3 +24,4 @@ export const RKEY_TS_FEES_PAID = 'reg:feesPaid:ts';
 
 export const RKEY_PACKAGE_SEARCHABLE = 'reg:packages';
 export const RKEY_ABI_SEARCHABLE = 'reg:abi';
+export const RKEY_REGISTRY_STREAM = 'reg:events';
diff --git a/packages/indexer/src/registry.ts b/packages/indexer/src/registry.ts
index b7761bdd9..e35d62281 100644
--- a/packages/indexer/src/registry.ts
+++ b/packages/indexer/src/registry.ts
@@ -508,6 +508,8 @@ export async function scanChain(
                 LABELS: { chainId: `${chainId}`, kind: rkey.RKEY_TS_FEES_PAID },
               });
 
+              batch.xAdd(rkey.RKEY_REGISTRY_STREAM, '*', event);
+
               batch.set(rkey.RKEY_LAST_UPDATED + ':' + event.chainId, event.timestamp);
 
               await batch.exec();

From 3dd71401277ebeebda76a19de3199d2d54b9c3a8 Mon Sep 17 00:00:00 2001
From: Daniel Beal <git@dbeal.dev>
Date: Sun, 4 Aug 2024 22:20:31 +0900
Subject: [PATCH 2/6] create ability to run auto verify

---
 examples/sample-foundry-project/lib/forge-std |   2 +-
 packages/builder/src/builder.ts               |   4 +-
 packages/builder/src/index.ts                 |   2 +-
 packages/indexer/src/auto-verify.ts           | 234 +++++++++++++++++-
 packages/indexer/src/config.ts                |   2 +
 packages/indexer/src/index.ts                 |   7 +-
 packages/indexer/src/redis.ts                 |   2 +
 7 files changed, 236 insertions(+), 17 deletions(-)

diff --git a/examples/sample-foundry-project/lib/forge-std b/examples/sample-foundry-project/lib/forge-std
index 52715a217..1d9650e95 160000
--- a/examples/sample-foundry-project/lib/forge-std
+++ b/examples/sample-foundry-project/lib/forge-std
@@ -1 +1 @@
-Subproject commit 52715a217dc51d0de15877878ab8213f6cbbbab5
+Subproject commit 1d9650e951204a0ddce9ff89c32f1997984cef4d
diff --git a/packages/builder/src/builder.ts b/packages/builder/src/builder.ts
index 8c85b8663..e9f1e4e4a 100644
--- a/packages/builder/src/builder.ts
+++ b/packages/builder/src/builder.ts
@@ -368,12 +368,12 @@ export function getArtifacts(def: ChainDefinition, state: DeploymentState) {
 }
 
 export async function getOutputs(
-  runtime: ChainBuilderRuntime,
+  runtime: ChainBuilderRuntime | null,
   def: ChainDefinition,
   state: DeploymentState
 ): Promise<ChainArtifacts | null> {
   const artifacts = getArtifacts(def, state);
-  if (runtime.snapshots) {
+  if (runtime?.snapshots) {
     // need to load state as well. the states that we want to load are the "leaf" layers
     const layers = _.uniq(Object.values(def.getStateLayers()));
 
diff --git a/packages/builder/src/index.ts b/packages/builder/src/index.ts
index 2ffd23265..c0fe72739 100644
--- a/packages/builder/src/index.ts
+++ b/packages/builder/src/index.ts
@@ -17,7 +17,7 @@ export { prepareMulticall } from './multicall';
 };
 
 export { CannonRegistry, OnChainRegistry, InMemoryRegistry, FallbackRegistry } from './registry';
-export { publishPackage, PackageReference, preparePublishPackage } from './package';
+export { publishPackage, forPackageTree, PackageReference, preparePublishPackage } from './package';
 export type { PackagePublishCall } from './package';
 export {
   CANNON_CHAIN_ID,
diff --git a/packages/indexer/src/auto-verify.ts b/packages/indexer/src/auto-verify.ts
index 2bf8f1353..57b0957e9 100644
--- a/packages/indexer/src/auto-verify.ts
+++ b/packages/indexer/src/auto-verify.ts
@@ -1,25 +1,237 @@
 import _ from 'lodash';
 import * as rkey from './db';
-import { ActualRedisClientType, useRedis } from './redis';
-import { verify } from '@usecannon/cli/dist/src/commands/verify';
-import { resolveCliSettings } from '@usecannon/cli/dist/src/settings';
+import { useRedis, commandOptions } from './redis';
+import { config } from './config';
+import {
+  forPackageTree,
+  DeploymentInfo,
+  ChainDefinition,
+  CannonStorage,
+  getOutputs,
+  IPFSLoader,
+  InMemoryRegistry,
+} from '@usecannon/builder';
+import viem from 'viem';
+import axios from 'axios';
 
 /* eslint no-console: "off" */
+interface EtherscanContract {
+  SourceCode: string;
+  ABI: string;
+  ContractName: string;
+  CompilerVersion: string;
+  OptimizationUsed: string;
+  Runs: string;
+  ConstructorArguments: string;
+  EVMVersion: string;
+  Library: string;
+  LicenseType: string;
+  Proxy: string;
+  Implementation: string;
+  SwarmSource: string;
+}
+
+interface EtherscanGetSourceCodeNotOkResponse {
+  status: '0';
+  message: 'NOTOK';
+  result: string;
+}
+
+interface EtherscanGetSourceCodeOkResponse {
+  status: '1';
+  message: 'OK';
+  result: EtherscanContract[];
+}
+
+// etherscan only supports some chain ids
+// https://docs.etherscan.io/contract-verification/supported-chains
+// for the chain ids not on this list, we should consider supporting blockscout api as needed in the future as well
+const SUPPORTED_CHAIN_IDS = [
+  1, 5, 11155111, 17000, 56, 97, 204, 5611, 250, 4002, 10, 420, 11155420, 137, 42161, 421614, 1284, 1287, 1285, 199, 1028,
+  42220, 44787, 100, 42170, 8453, 84532, 1101, 59144, 59140, 534352, 534351, 1111, 1112, 255, 2358, 252, 2522, 43114, 43113,
+  81457, 23888,
+];
+
+if (!config.ETHERSCAN_API_KEY) {
+  throw new Error('must specify CANNON_ETHERSCAN_API_KEY');
+}
+
+export type EtherscanGetSourceCodeResponse = EtherscanGetSourceCodeNotOkResponse | EtherscanGetSourceCodeOkResponse;
+
+export async function doContractVerify(ipfsHash: string, loader: CannonStorage) {
+  const guids: { [c: string]: string } = {};
+
+  const verifyPackage = async (deployData: DeploymentInfo) => {
+    if (SUPPORTED_CHAIN_IDS.includes(deployData.chainId || 0)) {
+      console.log('not verifying, unsupported chain id', deployData.chainId);
+      return {};
+    }
+
+    const miscData = await loader.readBlob(deployData.miscUrl);
+
+    const outputs = await getOutputs(null, new ChainDefinition(deployData.def), deployData.state);
+
+    if (!outputs) {
+      throw new Error('No chain outputs found. Has the requested chain already been built?');
+    }
+
+    for (const c in outputs.contracts) {
+      const contractInfo = outputs.contracts[c];
+
+      // contracts can either be imported by just their name, or by a full path.
+      // technically it may be more correct to just load by the actual name of the `artifact` property used, but that is complicated
+      console.log('finding contract:', contractInfo.sourceName, contractInfo.contractName);
+      const contractArtifact =
+        miscData.artifacts[contractInfo.contractName] ||
+        miscData.artifacts[`${contractInfo.sourceName}:${contractInfo.contractName}`];
+
+      if (!contractArtifact) {
+        console.log(`${c}: cannot verify: no contract artifact found`);
+        continue;
+      }
+
+      if (!contractArtifact.source) {
+        console.log(`${c}: cannot verify: no source code recorded in deploy data`);
+        continue;
+      }
+
+      if (await isVerified(contractInfo.address, config.ETHERSCAN_API_URL, config.ETHERSCAN_API_KEY)) {
+        console.log(`✅ ${c}: Contract source code already verified`);
+        await sleep(500);
+        continue;
+      }
+
+      try {
+        // supply any linked libraries within the inputs since those are calculated at runtime
+        const inputData = JSON.parse(contractArtifact.source.input);
+        inputData.settings.libraries = contractInfo.linkedLibraries;
+
+        const reqData: { [k: string]: string } = {
+          apikey: config.ETHERSCAN_API_KEY,
+          module: 'contract',
+          action: 'verifysourcecode',
+          chainId: '',
+          contractaddress: contractInfo.address,
+          // need to parse to get the inner structure, then stringify again
+          sourceCode: JSON.stringify(inputData),
+          codeformat: 'solidity-standard-json-input',
+          contractname: `${contractInfo.sourceName}:${contractInfo.contractName}`,
+          compilerversion: 'v' + contractArtifact.source.solcVersion,
+
+          // NOTE: below: yes, the etherscan api is misspelling
+          constructorArguements: viem
+            .encodeAbiParameters(
+              contractArtifact.abi.find((i: viem.AbiItem) => i.type === 'constructor')?.inputs ?? [],
+              contractInfo.constructorArgs || []
+            )
+            .slice(2),
+        };
+
+        const res = await axios.post(config.ETHERSCAN_API_KEY, reqData, {
+          headers: { 'content-type': 'application/x-www-form-urlencoded' },
+        });
+
+        if (res.data.status === '0') {
+          console.error(`${c}:\tcannot verify:`, res.data.result);
+        } else {
+          console.log(`${c}:\tsubmitted verification (${contractInfo.address})`);
+          guids[c] = res.data.result;
+        }
+      } catch (err) {
+        console.error(`verification for ${c} (${contractInfo.address}) failed:`, err);
+      }
+
+      await sleep(500);
+    }
+
+    return {};
+  };
+
+  const deployData = await loader.readBlob(ipfsHash);
+
+  if (!deployData) {
+    throw new Error(`loader could not load: ${ipfsHash}.`);
+  }
+
+  // go through all the packages and sub packages and make sure all contracts are being verified
+  await forPackageTree(loader, deployData, verifyPackage);
+}
+
+function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+/**
+ * Check if a smart contract is verified on Etherscan.
+ * @link https://docs.etherscan.io/api-endpoints/contracts#get-contract-source-code-for-verified-contract-source-codes
+ * @param address - The address of the smart contract.
+ * @param apiUrl - Etherscan API URL.
+ * @param apiKey - Etherscan API Key.
+ * @returns True if the contract is verified, false otherwise.
+ */
+
+export async function isVerified(address: string, apiUrl: string, apiKey: string): Promise<boolean> {
+  const parameters = new URLSearchParams({
+    apikey: apiKey,
+    module: 'contract',
+    action: 'getsourcecode',
+    address,
+  });
+
+  const url = new URL(apiUrl);
+  url.search = parameters.toString();
+
+  try {
+    const response = await fetch(url);
+
+    // checking that status is in the range 200-299 inclusive
+    if (!response.ok) {
+      throw new Error(`Network response failed (${response.status}: ${response.statusText})`);
+    }
+
+    const json = (await response.json()) as EtherscanGetSourceCodeResponse;
+
+    if (json.message !== 'OK') {
+      return false;
+    }
+
+    const sourceCode = json.result[0]?.SourceCode;
+    return sourceCode !== undefined && sourceCode !== null && sourceCode !== '';
+  } catch (e) {
+    return false;
+  }
+}
 
 export async function loop() {
-  const cliSettings = resolveCliSettings();
   const rdb = await useRedis();
 
-  const specialChainSettings = await import (process.cwd() + '/verify-chains.json');
+  let lastKey = await rdb.get(rkey.RKEY_REGISTRY_STREAM + ':auto-verify-last');
 
-  let lastKey = rdb.get(rkey.RKEY_REGISTRY_STREAM + ':auto-verify-last');
+  const loader = new CannonStorage(new InMemoryRegistry(), {
+    // shorter than usual timeout becuase we need to move on if its not resolving well
+    ipfs: new IPFSLoader(config.IPFS_URL, {}, 15000),
+  });
 
-  const e = await rdb.xRead(rkey.RKEY_REGISTRY_STREAM, { BLOCK: true, COUNT: 1 });
+  let e = await rdb.xRead(
+    commandOptions({ isolated: true }),
+    { key: rkey.RKEY_REGISTRY_STREAM, id: lastKey || '0-0' },
+    { BLOCK: 600, COUNT: 1 }
+  );
   while (e) {
-    // run the cannon verify command from the cli
-    await verify(e.packageUrl, _.defaults({ specialChainSettings[chainId.toString()] }, cliSettings), null, e.chainId);
+    try {
+      lastKey = e[0].messages[0].id;
+      const evt = e[0].messages[0].message;
+      // run the cannon verify command from the cli
+      await doContractVerify(evt.packageUrl, loader);
 
-    let lastKey = rdb.set(rkey.RKEY_REGISTRY_STREAM + ':auto-verify-last', ++lastKey);
-    e = await rdb.xRead(rkey.RKEY_REGISTRY_STREAM, { BLOCK: true, COUNT: 1 });
+      await rdb.set(rkey.RKEY_REGISTRY_STREAM + ':auto-verify-last', lastKey);
+      e = await rdb.xRead(
+        commandOptions({ isolated: true }),
+        { key: rkey.RKEY_REGISTRY_STREAM, id: lastKey || '0-0' },
+        { BLOCK: 600, COUNT: 1 }
+      );
+    } catch (err) {
+      console.error('during processing', err, e);
+    }
   }
 }
diff --git a/packages/indexer/src/config.ts b/packages/indexer/src/config.ts
index 59ed5298c..a1446b19d 100644
--- a/packages/indexer/src/config.ts
+++ b/packages/indexer/src/config.ts
@@ -12,4 +12,6 @@ export const config = cleanEnv(process.env, {
   NOTIFY_PKGS: str({ default: '' }),
   MAINNET_PROVIDER_URL: str({ default: 'https://ethereum-rpc.publicnode.com' }),
   OPTIMISM_PROVIDER_URL: str({ default: 'https://optimism-rpc.publicnode.com' }),
+  ETHERSCAN_API_URL: str({ default: 'https://api.etherscan.io/api' }),
+  ETHERSCAN_API_KEY: str({ default: '' }),
 });
diff --git a/packages/indexer/src/index.ts b/packages/indexer/src/index.ts
index 9b9ea55a0..404599d5a 100644
--- a/packages/indexer/src/index.ts
+++ b/packages/indexer/src/index.ts
@@ -1,4 +1,7 @@
-import { loop as registryLoop } from './registry';
 export * from './db';
 
-void registryLoop();
+import(`./${process.argv[2]}`)
+  .then((m) => void m.loop())
+  .catch((e) => {
+    throw e;
+  });
diff --git a/packages/indexer/src/redis.ts b/packages/indexer/src/redis.ts
index 4f910b20c..1d6ebf58e 100644
--- a/packages/indexer/src/redis.ts
+++ b/packages/indexer/src/redis.ts
@@ -1,6 +1,8 @@
 import { createClient } from 'redis';
 import { config } from './config';
 
+export { commandOptions } from 'redis';
+
 export type ActualRedisClientType = ReturnType<typeof createClient>;
 
 export async function useRedis() {

From d415089fe8c5989262c07f0bbaaa22ecc1b085a8 Mon Sep 17 00:00:00 2001
From: Daniel Beal <git@dbeal.dev>
Date: Fri, 9 Aug 2024 11:00:19 +0900
Subject: [PATCH 3/6] it works but we are stuck

---
 packages/indexer/src/auto-verify.ts | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/packages/indexer/src/auto-verify.ts b/packages/indexer/src/auto-verify.ts
index 57b0957e9..dd1baa61d 100644
--- a/packages/indexer/src/auto-verify.ts
+++ b/packages/indexer/src/auto-verify.ts
@@ -11,7 +11,7 @@ import {
   IPFSLoader,
   InMemoryRegistry,
 } from '@usecannon/builder';
-import viem from 'viem';
+import * as viem from 'viem';
 import axios from 'axios';
 
 /* eslint no-console: "off" */
@@ -53,7 +53,7 @@ const SUPPORTED_CHAIN_IDS = [
 ];
 
 if (!config.ETHERSCAN_API_KEY) {
-  throw new Error('must specify CANNON_ETHERSCAN_API_KEY');
+  throw new Error('must specify ETHERSCAN_API_KEY');
 }
 
 export type EtherscanGetSourceCodeResponse = EtherscanGetSourceCodeNotOkResponse | EtherscanGetSourceCodeOkResponse;
@@ -95,7 +95,7 @@ export async function doContractVerify(ipfsHash: string, loader: CannonStorage)
         continue;
       }
 
-      if (await isVerified(contractInfo.address, config.ETHERSCAN_API_URL, config.ETHERSCAN_API_KEY)) {
+      if (await isVerified(contractInfo.address, deployData.chainId!, config.ETHERSCAN_API_URL, config.ETHERSCAN_API_KEY)) {
         console.log(`✅ ${c}: Contract source code already verified`);
         await sleep(500);
         continue;
@@ -110,7 +110,7 @@ export async function doContractVerify(ipfsHash: string, loader: CannonStorage)
           apikey: config.ETHERSCAN_API_KEY,
           module: 'contract',
           action: 'verifysourcecode',
-          chainId: '',
+          chainId: deployData.chainId!.toString(),
           contractaddress: contractInfo.address,
           // need to parse to get the inner structure, then stringify again
           sourceCode: JSON.stringify(inputData),
@@ -127,7 +127,7 @@ export async function doContractVerify(ipfsHash: string, loader: CannonStorage)
             .slice(2),
         };
 
-        const res = await axios.post(config.ETHERSCAN_API_KEY, reqData, {
+        const res = await axios.post(config.ETHERSCAN_API_URL, reqData, {
           headers: { 'content-type': 'application/x-www-form-urlencoded' },
         });
 
@@ -170,11 +170,12 @@ function sleep(ms: number): Promise<void> {
  * @returns True if the contract is verified, false otherwise.
  */
 
-export async function isVerified(address: string, apiUrl: string, apiKey: string): Promise<boolean> {
+export async function isVerified(address: string, chainId: number, apiUrl: string, apiKey: string): Promise<boolean> {
   const parameters = new URLSearchParams({
     apikey: apiKey,
     module: 'contract',
     action: 'getsourcecode',
+    chainId: chainId.toString(),
     address,
   });
 

From 44cdcb1bdacf95e32f34196b21694bb3d894eafb Mon Sep 17 00:00:00 2001
From: Daniel Beal <git@dbeal.dev>
Date: Mon, 16 Sep 2024 00:45:35 +0900
Subject: [PATCH 4/6] fix bugs

---
 packages/indexer/package.json       |  1 +
 packages/indexer/src/auto-verify.ts | 16 +++++++++-------
 packages/indexer/src/config.ts      |  2 +-
 pnpm-lock.yaml                      | 18 ++++++++++++++++--
 4 files changed, 27 insertions(+), 10 deletions(-)

diff --git a/packages/indexer/package.json b/packages/indexer/package.json
index 82bb4b373..dde3c09ce 100644
--- a/packages/indexer/package.json
+++ b/packages/indexer/package.json
@@ -18,6 +18,7 @@
   "dependencies": {
     "@usecannon/builder": "workspace:*",
     "@usecannon/cli": "workspace:*",
+    "axios": "^1.7.7",
     "dotenv": "^16.4.5",
     "envalid": "^8.0.0",
     "lodash": "^4.17.21",
diff --git a/packages/indexer/src/auto-verify.ts b/packages/indexer/src/auto-verify.ts
index dd1baa61d..d2a75a53e 100644
--- a/packages/indexer/src/auto-verify.ts
+++ b/packages/indexer/src/auto-verify.ts
@@ -58,12 +58,13 @@ if (!config.ETHERSCAN_API_KEY) {
 
 export type EtherscanGetSourceCodeResponse = EtherscanGetSourceCodeNotOkResponse | EtherscanGetSourceCodeOkResponse;
 
-export async function doContractVerify(ipfsHash: string, loader: CannonStorage) {
+export async function doContractVerify(ipfsHash: string, loader: CannonStorage, chainId: number) {
   const guids: { [c: string]: string } = {};
 
   const verifyPackage = async (deployData: DeploymentInfo) => {
-    if (SUPPORTED_CHAIN_IDS.includes(deployData.chainId || 0)) {
-      console.log('not verifying, unsupported chain id', deployData.chainId);
+    if (!SUPPORTED_CHAIN_IDS.includes(chainId)) {
+      console.log('not verifying, unsupported chain id', chainId);
+      console.log('SUPPORTED', SUPPORTED_CHAIN_IDS);
       return {};
     }
 
@@ -95,7 +96,7 @@ export async function doContractVerify(ipfsHash: string, loader: CannonStorage)
         continue;
       }
 
-      if (await isVerified(contractInfo.address, deployData.chainId!, config.ETHERSCAN_API_URL, config.ETHERSCAN_API_KEY)) {
+      if (await isVerified(contractInfo.address, chainId, config.ETHERSCAN_API_URL, config.ETHERSCAN_API_KEY)) {
         console.log(`✅ ${c}: Contract source code already verified`);
         await sleep(500);
         continue;
@@ -110,7 +111,7 @@ export async function doContractVerify(ipfsHash: string, loader: CannonStorage)
           apikey: config.ETHERSCAN_API_KEY,
           module: 'contract',
           action: 'verifysourcecode',
-          chainId: deployData.chainId!.toString(),
+          chainid: chainId!.toString(),
           contractaddress: contractInfo.address,
           // need to parse to get the inner structure, then stringify again
           sourceCode: JSON.stringify(inputData),
@@ -175,7 +176,7 @@ export async function isVerified(address: string, chainId: number, apiUrl: strin
     apikey: apiKey,
     module: 'contract',
     action: 'getsourcecode',
-    chainId: chainId.toString(),
+    chainid: chainId.toString(),
     address,
   });
 
@@ -222,8 +223,9 @@ export async function loop() {
     try {
       lastKey = e[0].messages[0].id;
       const evt = e[0].messages[0].message;
+      const chainId = parseInt(evt.variant.split('-')[0]);
       // run the cannon verify command from the cli
-      await doContractVerify(evt.packageUrl, loader);
+      await doContractVerify(evt.packageUrl, loader, chainId);
 
       await rdb.set(rkey.RKEY_REGISTRY_STREAM + ':auto-verify-last', lastKey);
       e = await rdb.xRead(
diff --git a/packages/indexer/src/config.ts b/packages/indexer/src/config.ts
index a1446b19d..4c16a7c01 100644
--- a/packages/indexer/src/config.ts
+++ b/packages/indexer/src/config.ts
@@ -12,6 +12,6 @@ export const config = cleanEnv(process.env, {
   NOTIFY_PKGS: str({ default: '' }),
   MAINNET_PROVIDER_URL: str({ default: 'https://ethereum-rpc.publicnode.com' }),
   OPTIMISM_PROVIDER_URL: str({ default: 'https://optimism-rpc.publicnode.com' }),
-  ETHERSCAN_API_URL: str({ default: 'https://api.etherscan.io/api' }),
+  ETHERSCAN_API_URL: str({ default: 'https://api.etherscan.io/v2/api' }),
   ETHERSCAN_API_KEY: str({ default: '' }),
 });
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 852497861..a030f377b 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -505,6 +505,9 @@ importers:
       '@usecannon/cli':
         specifier: workspace:*
         version: link:../cli
+      axios:
+        specifier: ^1.7.7
+        version: 1.7.7
       dotenv:
         specifier: ^16.4.5
         version: 16.4.5
@@ -5849,6 +5852,9 @@ packages:
   axios@1.7.5:
     resolution: {integrity: sha512-fZu86yCo+svH3uqJ/yTdQ0QHpQu5oL+/QE+QPSv6BZSkDAoky9vytxp7u5qk83OJFS3kEBcesWni9WTZAv3tSw==}
 
+  axios@1.7.7:
+    resolution: {integrity: sha512-S4kL7XrjgBmvdGut0sN3yJxqYzrDOnivkBiN0OFs6hLiUam3UPvswUo0kqGyhqUZGEOytHyumEdXsAkgCOUf3Q==}
+
   axobject-query@3.1.1:
     resolution: {integrity: sha512-goKlv8DZrK9hUh975fnHzhNIO4jUnFCfv/dszV5VwUGDFjI6vQ2VwoyjYjYNEbBE8AH87TduWP5uyDR1D+Iteg==}
 
@@ -15900,7 +15906,7 @@ snapshots:
       '@cucumber/ci-environment': 10.0.1
       '@cucumber/cucumber-expressions': 17.1.0
       '@cucumber/gherkin': 28.0.0
-      '@cucumber/gherkin-streams': 5.0.1(@cucumber/gherkin@28.0.0)(@cucumber/message-streams@4.0.1(@cucumber/messages@25.0.1))(@cucumber/messages@24.1.0)
+      '@cucumber/gherkin-streams': 5.0.1(@cucumber/gherkin@28.0.0)(@cucumber/message-streams@4.0.1(@cucumber/messages@24.1.0))(@cucumber/messages@24.1.0)
       '@cucumber/gherkin-utils': 9.0.0
       '@cucumber/html-formatter': 21.6.0(@cucumber/messages@24.1.0)
       '@cucumber/message-streams': 4.0.1(@cucumber/messages@24.1.0)
@@ -15940,7 +15946,7 @@ snapshots:
       yaml: 2.5.0
       yup: 1.2.0
 
-  '@cucumber/gherkin-streams@5.0.1(@cucumber/gherkin@28.0.0)(@cucumber/message-streams@4.0.1(@cucumber/messages@25.0.1))(@cucumber/messages@24.1.0)':
+  '@cucumber/gherkin-streams@5.0.1(@cucumber/gherkin@28.0.0)(@cucumber/message-streams@4.0.1(@cucumber/messages@24.1.0))(@cucumber/messages@24.1.0)':
     dependencies:
       '@cucumber/gherkin': 28.0.0
       '@cucumber/message-streams': 4.0.1(@cucumber/messages@24.1.0)
@@ -20959,6 +20965,14 @@ snapshots:
     transitivePeerDependencies:
       - debug
 
+  axios@1.7.7:
+    dependencies:
+      follow-redirects: 1.15.6(debug@4.3.6)
+      form-data: 4.0.0
+      proxy-from-env: 1.1.0
+    transitivePeerDependencies:
+      - debug
+
   axobject-query@3.1.1:
     dependencies:
       deep-equal: 2.2.3

From c163338050c8fbbc74c93ecf2091df93fc178b76 Mon Sep 17 00:00:00 2001
From: Daniel Beal <git@dbeal.dev>
Date: Mon, 16 Sep 2024 12:47:57 +0900
Subject: [PATCH 5/6] registry handling fixes

---
 package.json                     |  1 +
 packages/indexer/Dockerfile      | 46 +++++++++++++++++++++-----------
 packages/indexer/src/db.ts       |  1 +
 packages/indexer/src/registry.ts | 10 +++++--
 4 files changed, 41 insertions(+), 17 deletions(-)

diff --git a/package.json b/package.json
index a418ee833..4f9c2b385 100644
--- a/package.json
+++ b/package.json
@@ -10,6 +10,7 @@
     "lint:fix:sol": "prettier --write '{packages,examples}/*/{contracts,src}/**/*.sol'",
     "lint:fix": "pnpm run '/^lint:fix:(js|sol)/'",
     "build": "pnpm -r --filter @usecannon/builder --filter @usecannon/cli --filter hardhat-cannon --filter @usecannon/api run build",
+    "docker-indexer": "docker build --platform=linux/amd64,linux/arm64 . -f packages/indexer/Dockerfile -t ghcr.io/usecannon/cannon/indexer:$(jq -r '.version' lerna.json)",
     "watch": "pnpm -r --parallel --filter @usecannon/builder --filter @usecannon/cli run watch",
     "version-alpha": "lerna version prerelease --no-private",
     "version-patch": "lerna version patch --no-private",
diff --git a/packages/indexer/Dockerfile b/packages/indexer/Dockerfile
index 3cd8e42fe..0e2f9fee2 100644
--- a/packages/indexer/Dockerfile
+++ b/packages/indexer/Dockerfile
@@ -1,16 +1,32 @@
-FROM node:18-alpine AS build
-WORKDIR /app
-ENV REDIS_URL ""
-ENV IPFS_URL ""
-ENV MAINNET_PROVIDER_URL ""
-COPY --link . .
-RUN npm i
-RUN npm run build
+FROM node:18-alpine AS base
 
-FROM node:18-alpine
-WORKDIR /app
-COPY --link --from=build /app/dist dist
-COPY --link --from=build /app/package.json package.json
-COPY --link --from=build /app/package-lock.json package-lock.json
-RUN npm install --omit=dev
-CMD ["node", "dist/index.js"]
+# app envvars
+ENV REDIS_URL="" IPFS_URL="" MAINNET_PROVIDER_URL=""
+
+# pnpm envvars
+ENV PNPM_HOME="/pnpm"
+ENV PATH="$PNPM_HOME:$PATH"
+RUN corepack enable
+
+COPY . /cannon
+
+WORKDIR /cannon
+
+FROM base AS prod-deps
+# for some reason some packages require python and make (?) to finish their installation (?)
+RUN apk update && apk add python3 build-base
+RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install --prod --frozen-lockfile
+
+FROM base AS build
+# for some reason some packages require python and make (?) to finish their installation (?)
+RUN apk update && apk add python3 build-base
+RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install --frozen-lockfile
+RUN pnpm run build
+RUN cd packages/indexer && pnpm run build
+
+FROM base
+COPY --from=prod-deps /cannon/node_modules /cannon/node_modules
+COPY --from=build /cannon/packages/builder/dist /cannon/packages/builder/dist
+COPY --from=build /cannon/packages/cli/dist /cannon/packages/cli/dist
+COPY --from=build /cannon/packages/indexer/dist /cannon/packages/indexer/dist
+CMD ["node", "packages/indexer/dist/index.js"]
diff --git a/packages/indexer/src/db.ts b/packages/indexer/src/db.ts
index 6cb724b0e..15edd58f6 100644
--- a/packages/indexer/src/db.ts
+++ b/packages/indexer/src/db.ts
@@ -1,5 +1,6 @@
 export const RKEY_LAST_IDX = 'reg:lastBlock';
 export const RKEY_LAST_UPDATED = 'reg:lastTimestamp';
+export const RKEY_START_SYNC = 'reg:startSync';
 export const RKEY_ADDRESS_TO_PACKAGE = 'reg:addressToPackage';
 export const RKEY_TRANSACTION_TO_PACKAGE = 'reg:transactionToPackage';
 export const RKEY_PACKAGE_RELATION = 'reg:packageRelation';
diff --git a/packages/indexer/src/registry.ts b/packages/indexer/src/registry.ts
index e35d62281..16d25ddc4 100644
--- a/packages/indexer/src/registry.ts
+++ b/packages/indexer/src/registry.ts
@@ -467,6 +467,13 @@ export async function scanChain(
           const feePaid = event.args.feePaid || 0n;
 
           const batch = redis.multi();
+
+          // add an event sourcable record to the index
+          batch.xAdd(`${rkey.RKEY_REGISTRY_STREAM}:${event.chainId}`, `${event.timestamp * 1000}-*`, {
+            event: 'PackagePublish',
+            ..._.mapValues(event.args, (v) => v.toString()),
+          });
+
           switch (event.eventName) {
             case 'PackagePublish':
             case 'PackagePublishWithFee':
@@ -508,8 +515,6 @@ export async function scanChain(
                 LABELS: { chainId: `${chainId}`, kind: rkey.RKEY_TS_FEES_PAID },
               });
 
-              batch.xAdd(rkey.RKEY_REGISTRY_STREAM, '*', event);
-
               batch.set(rkey.RKEY_LAST_UPDATED + ':' + event.chainId, event.timestamp);
 
               await batch.exec();
@@ -583,6 +588,7 @@ export async function scanChain(
 
       await redis.set(rkey.RKEY_LAST_IDX + ':' + 1, mainnetScan.scanToBlock);
       await redis.set(rkey.RKEY_LAST_IDX + ':' + 10, optimismScan.scanToBlock);
+      await redis.set(rkey.RKEY_START_SYNC, Math.floor(Date.now() / 1000), { NX: true });
       consecutiveFailures = 0;
     } catch (err) {
       console.error('failure while scanning cannon publishes:', err);

From a33675c839e4db86e8554a6093eab4bf461a00e9 Mon Sep 17 00:00:00 2001
From: Daniel Beal <git@dbeal.dev>
Date: Mon, 16 Sep 2024 12:48:33 +0900
Subject: [PATCH 6/6] add missing dockerignore

---
 packages/indexer/.dockerignore | 5 +++++
 1 file changed, 5 insertions(+)
 create mode 100644 packages/indexer/.dockerignore

diff --git a/packages/indexer/.dockerignore b/packages/indexer/.dockerignore
new file mode 100644
index 000000000..4f75c9513
--- /dev/null
+++ b/packages/indexer/.dockerignore
@@ -0,0 +1,5 @@
+node_modules
+.git
+.gitignore
+*.md
+dist