etherscan contracts auto-verify

package.json

@@ -10,6 +10,7 @@
     "lint:fix:sol": "prettier --write '{packages,examples}/*/{contracts,src}/**/*.sol'",
     "lint:fix": "pnpm run '/^lint:fix:(js|sol)/'",
     "build": "pnpm -r --filter @usecannon/builder --filter @usecannon/cli --filter hardhat-cannon --filter @usecannon/api run build",
+    "docker-indexer": "docker build --platform=linux/amd64,linux/arm64 . -f packages/indexer/Dockerfile -t ghcr.io/usecannon/cannon/indexer:$(jq -r '.version' lerna.json)",
     "watch": "pnpm -r --parallel --filter @usecannon/builder --filter @usecannon/cli run watch",
     "version-alpha": "lerna version prerelease --no-private",
     "version-patch": "lerna version patch --no-private",

packages/builder/src/builder.ts

@@ -367,12 +367,12 @@ export function getArtifacts(def: ChainDefinition, state: DeploymentState) {
 }
 
 export async function getOutputs(
-  runtime: ChainBuilderRuntime,
+  runtime: ChainBuilderRuntime | null,
   def: ChainDefinition,
   state: DeploymentState
 ): Promise<ChainArtifacts> {
   const artifacts = getArtifacts(def, state);
-  if (runtime.snapshots) {
+  if (runtime?.snapshots) {
     // need to load state as well. the states that we want to load are the "leaf" layers
     const layers = _.uniq(Object.values(def.getStateLayers()));
 

packages/builder/src/index.ts

@@ -17,7 +17,7 @@ export { prepareMulticall } from './multicall';
 };
 
 export { CannonRegistry, OnChainRegistry, InMemoryRegistry, FallbackRegistry } from './registry';
-export { publishPackage, preparePublishPackage, forPackageTree } from './package';
+export { publishPackage, forPackageTree, preparePublishPackage } from './package';
 export type { PackagePublishCall } from './package';
 export {
   CANNON_CHAIN_ID,

packages/indexer/.dockerignore

@@ -0,0 +1,5 @@
+node_modules
+.git
+.gitignore
+*.md
+dist

packages/indexer/Dockerfile

@@ -1,16 +1,32 @@
-FROM node:18-alpine AS build
-WORKDIR /app
-ENV REDIS_URL ""
-ENV IPFS_URL ""
-ENV MAINNET_PROVIDER_URL ""
-COPY --link . .
-RUN npm i
-RUN npm run build
+FROM node:18-alpine AS base
 
-FROM node:18-alpine
-WORKDIR /app
-COPY --link --from=build /app/dist dist
-COPY --link --from=build /app/package.json package.json
-COPY --link --from=build /app/package-lock.json package-lock.json
-RUN npm install --omit=dev
-CMD ["node", "dist/index.js"]
+# app envvars
+ENV REDIS_URL="" IPFS_URL="" MAINNET_PROVIDER_URL=""
+
+# pnpm envvars
+ENV PNPM_HOME="/pnpm"
+ENV PATH="$PNPM_HOME:$PATH"
+RUN corepack enable
+
+COPY . /cannon
+
+WORKDIR /cannon
+
+FROM base AS prod-deps
+# for some reason some packages require python and make (?) to finish their installation (?)
+RUN apk update && apk add python3 build-base
+RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install --prod --frozen-lockfile
+
+FROM base AS build
+# for some reason some packages require python and make (?) to finish their installation (?)
+RUN apk update && apk add python3 build-base
+RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install --frozen-lockfile
+RUN pnpm run build
+RUN cd packages/indexer && pnpm run build
+
+FROM base
+COPY --from=prod-deps /cannon/node_modules /cannon/node_modules
+COPY --from=build /cannon/packages/builder/dist /cannon/packages/builder/dist
+COPY --from=build /cannon/packages/cli/dist /cannon/packages/cli/dist
+COPY --from=build /cannon/packages/indexer/dist /cannon/packages/indexer/dist
+CMD ["node", "packages/indexer/dist/index.js"]

packages/indexer/package.json

@@ -18,6 +18,7 @@
   "dependencies": {
     "@usecannon/builder": "workspace:*",
     "@usecannon/cli": "workspace:*",
+    "axios": "^1.7.7",
     "dotenv": "^16.4.5",
     "envalid": "^8.0.0",
     "lodash": "^4.17.21",

packages/indexer/src/auto-verify.ts

@@ -0,0 +1,240 @@
+import _ from 'lodash';
+import * as rkey from './db';
+import { useRedis, commandOptions } from './redis';
+import { config } from './config';
+import {
+  forPackageTree,
+  DeploymentInfo,
+  ChainDefinition,
+  CannonStorage,
+  getOutputs,
+  IPFSLoader,
+  InMemoryRegistry,
+} from '@usecannon/builder';
+import * as viem from 'viem';
+import axios from 'axios';
+
+/* eslint no-console: "off" */
+interface EtherscanContract {
+  SourceCode: string;
+  ABI: string;
+  ContractName: string;
+  CompilerVersion: string;
+  OptimizationUsed: string;
+  Runs: string;
+  ConstructorArguments: string;
+  EVMVersion: string;
+  Library: string;
+  LicenseType: string;
+  Proxy: string;
+  Implementation: string;
+  SwarmSource: string;
+}
+
+interface EtherscanGetSourceCodeNotOkResponse {
+  status: '0';
+  message: 'NOTOK';
+  result: string;
+}
+
+interface EtherscanGetSourceCodeOkResponse {
+  status: '1';
+  message: 'OK';
+  result: EtherscanContract[];
+}
+
+// etherscan only supports some chain ids
+// https://docs.etherscan.io/contract-verification/supported-chains
+// for the chain ids not on this list, we should consider supporting blockscout api as needed in the future as well
+const SUPPORTED_CHAIN_IDS = [
+  1, 5, 11155111, 17000, 56, 97, 204, 5611, 250, 4002, 10, 420, 11155420, 137, 42161, 421614, 1284, 1287, 1285, 199, 1028,
+  42220, 44787, 100, 42170, 8453, 84532, 1101, 59144, 59140, 534352, 534351, 1111, 1112, 255, 2358, 252, 2522, 43114, 43113,
+  81457, 23888,
+];
+
+if (!config.ETHERSCAN_API_KEY) {
+  throw new Error('must specify ETHERSCAN_API_KEY');
+}
+
+export type EtherscanGetSourceCodeResponse = EtherscanGetSourceCodeNotOkResponse | EtherscanGetSourceCodeOkResponse;
+
+export async function doContractVerify(ipfsHash: string, loader: CannonStorage, chainId: number) {
+  const guids: { [c: string]: string } = {};
+
+  const verifyPackage = async (deployData: DeploymentInfo) => {
+    if (!SUPPORTED_CHAIN_IDS.includes(chainId)) {
+      console.log('not verifying, unsupported chain id', chainId);
+      console.log('SUPPORTED', SUPPORTED_CHAIN_IDS);
+      return {};
+    }
+
+    const miscData = await loader.readBlob(deployData.miscUrl);
+
+    const outputs = await getOutputs(null, new ChainDefinition(deployData.def), deployData.state);
+
+    if (!outputs) {
+      throw new Error('No chain outputs found. Has the requested chain already been built?');
+    }
+
+    for (const c in outputs.contracts) {
+      const contractInfo = outputs.contracts[c];
+
+      // contracts can either be imported by just their name, or by a full path.
+      // technically it may be more correct to just load by the actual name of the `artifact` property used, but that is complicated
+      console.log('finding contract:', contractInfo.sourceName, contractInfo.contractName);
+      const contractArtifact =
+        miscData.artifacts[contractInfo.contractName] ||
+        miscData.artifacts[`${contractInfo.sourceName}:${contractInfo.contractName}`];
+
+      if (!contractArtifact) {
+        console.log(`${c}: cannot verify: no contract artifact found`);
+        continue;
+      }
+
+      if (!contractArtifact.source) {
+        console.log(`${c}: cannot verify: no source code recorded in deploy data`);
+        continue;
+      }
+
+      if (await isVerified(contractInfo.address, chainId, config.ETHERSCAN_API_URL, config.ETHERSCAN_API_KEY)) {
+        console.log(`✅ ${c}: Contract source code already verified`);
+        await sleep(500);
+        continue;
+      }
+
+      try {
+        // supply any linked libraries within the inputs since those are calculated at runtime
+        const inputData = JSON.parse(contractArtifact.source.input);
+        inputData.settings.libraries = contractInfo.linkedLibraries;
+
+        const reqData: { [k: string]: string } = {
+          apikey: config.ETHERSCAN_API_KEY,
+          module: 'contract',
+          action: 'verifysourcecode',
+          chainid: chainId!.toString(),
+          contractaddress: contractInfo.address,
+          // need to parse to get the inner structure, then stringify again
+          sourceCode: JSON.stringify(inputData),
+          codeformat: 'solidity-standard-json-input',
+          contractname: `${contractInfo.sourceName}:${contractInfo.contractName}`,
+          compilerversion: 'v' + contractArtifact.source.solcVersion,
+
+          // NOTE: below: yes, the etherscan api is misspelling
+          constructorArguements: viem
+            .encodeAbiParameters(
+              contractArtifact.abi.find((i: viem.AbiItem) => i.type === 'constructor')?.inputs ?? [],
+              contractInfo.constructorArgs || []
+            )
+            .slice(2),
+        };
+
+        const res = await axios.post(config.ETHERSCAN_API_URL, reqData, {
+          headers: { 'content-type': 'application/x-www-form-urlencoded' },
+        });
+
+        if (res.data.status === '0') {
+          console.error(`${c}:\tcannot verify:`, res.data.result);
+        } else {
+          console.log(`${c}:\tsubmitted verification (${contractInfo.address})`);
+          guids[c] = res.data.result;
+        }
+      } catch (err) {
+        console.error(`verification for ${c} (${contractInfo.address}) failed:`, err);
+      }
+
+      await sleep(500);
+    }
+
+    return {};
+  };
+
+  const deployData = await loader.readBlob(ipfsHash);
+
+  if (!deployData) {
+    throw new Error(`loader could not load: ${ipfsHash}.`);
+  }
+
+  // go through all the packages and sub packages and make sure all contracts are being verified
+  await forPackageTree(loader, deployData, verifyPackage);
+}
+
+function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+/**
+ * Check if a smart contract is verified on Etherscan.
+ * @link https://docs.etherscan.io/api-endpoints/contracts#get-contract-source-code-for-verified-contract-source-codes
+ * @param address - The address of the smart contract.
+ * @param apiUrl - Etherscan API URL.
+ * @param apiKey - Etherscan API Key.
+ * @returns True if the contract is verified, false otherwise.
+ */
+
+export async function isVerified(address: string, chainId: number, apiUrl: string, apiKey: string): Promise<boolean> {
+  const parameters = new URLSearchParams({
+    apikey: apiKey,
+    module: 'contract',
+    action: 'getsourcecode',
+    chainid: chainId.toString(),
+    address,
+  });
+
+  const url = new URL(apiUrl);
+  url.search = parameters.toString();
+
+  try {
+    const response = await fetch(url);
+
+    // checking that status is in the range 200-299 inclusive
+    if (!response.ok) {
+      throw new Error(`Network response failed (${response.status}: ${response.statusText})`);
+    }
+
+    const json = (await response.json()) as EtherscanGetSourceCodeResponse;
+
+    if (json.message !== 'OK') {
+      return false;
+    }
+
+    const sourceCode = json.result[0]?.SourceCode;
+    return sourceCode !== undefined && sourceCode !== null && sourceCode !== '';
+  } catch (e) {
+    return false;
+  }
+}
+
+export async function loop() {
+  const rdb = await useRedis();
+
+  let lastKey = await rdb.get(rkey.RKEY_REGISTRY_STREAM + ':auto-verify-last');
+
+  const loader = new CannonStorage(new InMemoryRegistry(), {
+    // shorter than usual timeout becuase we need to move on if its not resolving well
+    ipfs: new IPFSLoader(config.IPFS_URL, {}, 15000),
+  });
+
+  let e = await rdb.xRead(
+    commandOptions({ isolated: true }),
+    { key: rkey.RKEY_REGISTRY_STREAM, id: lastKey || '0-0' },
+    { BLOCK: 600, COUNT: 1 }
+  );
+  while (e) {
+    try {
+      lastKey = e[0].messages[0].id;
+      const evt = e[0].messages[0].message;
+      const chainId = parseInt(evt.variant.split('-')[0]);
+      // run the cannon verify command from the cli
+      await doContractVerify(evt.packageUrl, loader, chainId);
+
+      await rdb.set(rkey.RKEY_REGISTRY_STREAM + ':auto-verify-last', lastKey);
+      e = await rdb.xRead(
+        commandOptions({ isolated: true }),
+        { key: rkey.RKEY_REGISTRY_STREAM, id: lastKey || '0-0' },
+        { BLOCK: 600, COUNT: 1 }
+      );
+    } catch (err) {
+      console.error('during processing', err, e);
+    }
+  }
+}

packages/indexer/src/config.ts

@@ -12,4 +12,6 @@ export const config = cleanEnv(process.env, {
   NOTIFY_PKGS: str({ default: '' }),
   MAINNET_PROVIDER_URL: str({ default: 'https://ethereum-rpc.publicnode.com' }),
   OPTIMISM_PROVIDER_URL: str({ default: 'https://optimism-rpc.publicnode.com' }),
+  ETHERSCAN_API_URL: str({ default: 'https://api.etherscan.io/v2/api' }),
+  ETHERSCAN_API_KEY: str({ default: '' }),
 });

packages/indexer/src/db.ts

@@ -1,5 +1,6 @@
 export const RKEY_LAST_IDX = 'reg:lastBlock';
 export const RKEY_LAST_UPDATED = 'reg:lastTimestamp';
+export const RKEY_START_SYNC = 'reg:startSync';
 export const RKEY_ADDRESS_TO_PACKAGE = 'reg:addressToPackage';
 export const RKEY_TRANSACTION_TO_PACKAGE = 'reg:transactionToPackage';
 export const RKEY_PACKAGE_RELATION = 'reg:packageRelation';
@@ -24,3 +25,4 @@ export const RKEY_TS_FEES_PAID = 'reg:feesPaid:ts';
 
 export const RKEY_PACKAGE_SEARCHABLE = 'reg:packages';
 export const RKEY_ABI_SEARCHABLE = 'reg:abi';
+export const RKEY_REGISTRY_STREAM = 'reg:events';

packages/indexer/src/index.ts

@@ -1,4 +1,7 @@
-import { loop as registryLoop } from './registry';
 export * from './db';
 
-void registryLoop();
+import(`./${process.argv[2]}`)
+  .then((m) => void m.loop())
+  .catch((e) => {
+    throw e;
+  });

packages/indexer/src/redis.ts

@@ -1,6 +1,8 @@
 import { createClient } from 'redis';
 import { config } from './config';
 
+export { commandOptions } from 'redis';
+
 export type ActualRedisClientType = ReturnType<typeof createClient>;
 
 export async function useRedis(): Promise<ActualRedisClientType> {

packages/indexer/src/registry.ts

@@ -467,6 +467,13 @@ export async function scanChain(
           const feePaid = event.args.feePaid || 0n;
 
           const batch = redis.multi();
+
+          // add an event sourcable record to the index
+          batch.xAdd(`${rkey.RKEY_REGISTRY_STREAM}:${event.chainId}`, `${event.timestamp * 1000}-*`, {
+            event: 'PackagePublish',
+            ..._.mapValues(event.args, (v) => v.toString()),
+          });
+
           switch (event.eventName) {
             case 'PackagePublish':
             case 'PackagePublishWithFee':
@@ -581,6 +588,7 @@ export async function scanChain(
 
       await redis.set(rkey.RKEY_LAST_IDX + ':' + 1, mainnetScan.scanToBlock);
       await redis.set(rkey.RKEY_LAST_IDX + ':' + 10, optimismScan.scanToBlock);
+      await redis.set(rkey.RKEY_START_SYNC, Math.floor(Date.now() / 1000), { NX: true });
       consecutiveFailures = 0;
     } catch (err) {
       console.error('failure while scanning cannon publishes:', err);