feat: crawl evm block merge develop

ci/config.json.ci

@@ -424,5 +424,26 @@
   "jobSyncSourcify": {
     "millisecondCrawl": 2000,
     "recordsPerCall": 100
+  },
+  "crawlEvmBlock": {
+    "millisecondCrawl": 5000,
+    "blocksPerCall": 100,
+    "startBlock": 19500000,
+    "saveRawLog": true
+  },
+  "createEvmBlockToPartition": {
+    "startId": 0,
+    "endId": 100000000,
+    "step": 100000000
+  },
+  "jobCheckNeedCreateEVMBlockPartition": {
+    "millisecondCrawl": 10000,
+    "templateTable": "evm_block"
+  },
+  "jobCreateConstraintInEVMBlockPartition": {
+    "jobRepeatCheckNeedCreateConstraint": {
+      "millisecondRepeatJob": 10000
+    },
+    "statementTimeout": 600000
   }
 }

config.json

@@ -422,5 +422,26 @@
   "jobSyncSourcify": {
     "millisecondCrawl": 2000,
     "recordsPerCall": 100
+  },
+  "crawlEvmBlock": {
+    "millisecondCrawl": 5000,
+    "blocksPerCall": 100,
+    "startBlock": 19500000,
+    "saveRawLog": true
+  },
+  "createEvmBlockToPartition": {
+    "startId": 0,
+    "endId": 100000000,
+    "step": 100000000
+  },
+  "jobCheckNeedCreateEVMBlockPartition": {
+    "millisecondCrawl": 10000,
+    "templateTable": "evm_block"
+  },
+  "jobCreateConstraintInEVMBlockPartition": {
+    "jobRepeatCheckNeedCreateConstraint": {
+      "millisecondRepeatJob": 10000
+    },
+    "statementTimeout": 600000
   }
 }

migrations/evm/20240517090021_create_evm_block.ts

@@ -0,0 +1,59 @@
+import { Knex } from 'knex';
+import config from '../../config.json' assert { type: 'json' };
+
+export async function up(knex: Knex): Promise<void> {
+  await knex.raw(
+    `CREATE TABLE evm_block(
+      height integer,
+      tx_count integer,
+      base_fee_per_gas numeric(80,0),
+      date timestamp with time zone,
+      difficulty numeric(80,0),
+      extra_data text,
+      gas_limit numeric(80,0),
+      gas_used numeric(80,0),
+      hash text,
+      miner text,
+      nonce text,
+      parent_hash text,
+      receipts_root text,
+      state_root text,
+      transactions jsonb
+    ) PARTITION BY RANGE(height);
+    CREATE INDEX evm_blockheight_index ON evm_block(height);
+    CREATE INDEX evm_block_hash_index ON evm_block(hash);
+    CREATE INDEX evm_block_date_index ON evm_block(date);`
+  );
+  let endId = config.createEvmBlockToPartition.endId;
+  let step = config.createEvmBlockToPartition.step;
+  for (let i = 0; i < endId; i += step) {
+    const partitionName = `evm_block_partition_${i}_${i + step}`;
+    await knex.raw(
+      `CREATE TABLE ${partitionName} (LIKE evm_block INCLUDING ALL)`
+    );
+    await knex.raw(
+      `ALTER TABLE evm_block ATTACH PARTITION ${partitionName} FOR VALUES FROM (${i}) to (${
+        i + step
+      })`
+    );
+  }
+}
+
+export async function down(knex: Knex): Promise<void> {
+  const listTablePartition: any = await knex.raw(
+    `SELECT
+    parent.relname      AS parent,
+    child.relname       AS child,
+  pg_get_expr(child.relpartbound, child.oid) AS bounds
+  FROM pg_inherits
+      JOIN pg_class parent            ON pg_inherits.inhparent = parent.oid
+      JOIN pg_class child             ON pg_inherits.inhrelid   = child.oid
+  WHERE parent.relname='evm_block';`
+  );
+  await Promise.all(
+    listTablePartition.rows.map((partition: any) => {
+      return knex.raw(`DROP TABLE ${partition.child}`);
+    })
+  );
+  await knex.raw('DROP TABLE evm_block');
+}

src/models/evm_block.ts

@@ -0,0 +1,63 @@
+import BaseModel from './base';
+
+export class EVMBlock extends BaseModel {
+  height!: number;
+
+  tx_count!: number;
+
+  base_fee_per_gas!: bigint;
+
+  date!: Date;
+
+  difficulty!: bigint;
+
+  extra_data!: string;
+
+  gas_limit!: bigint;
+
+  gas_used!: bigint;
+
+  hash!: string;
+
+  miner!: string;
+
+  nonce!: string;
+
+  parent_hash!: string;
+
+  receipts_root!: string;
+
+  state_root!: string;
+
+  transactions!: any;
+
+  static get tableName() {
+    return 'evm_block';
+  }
+
+  static get idColumn() {
+    return 'height';
+  }
+
+  static get jsonSchema() {
+    return {
+      type: 'object',
+      required: [
+        'height',
+        'hash',
+        'tx_count',
+        'parent_hash',
+        'receipts_root',
+        'state_root',
+      ],
+      properties: {
+        required: { type: 'number' },
+        tx_count: { type: 'number' },
+        hash: { type: 'string' },
+        parent_hash: { type: 'string' },
+        receipts_root: { type: 'string' },
+        state_root: { type: 'string' },
+      },
+    };
+  }
+}

src/models/index.ts

@@ -45,3 +45,4 @@ export * from './erc721_token';
 export * from './account_balance';
 export * from './erc721_contract';
 export * from './erc721_stats';
+export * from './evm_block';

src/services/evm/constant.ts

@@ -175,6 +175,18 @@ export const SERVICE = {
           path: 'v1.InsertVerifyByCodeHash.createJobInsertVerifyByCodeHash',
         },
       },
+      CreateEVMBlockPartition: {
+        key: 'CreateEVMBlockPartition',
+        path: 'v1.CreateEVMBlockPartition',
+      },
+      CreateConstraintInEVMBlockPartition: {
+        key: 'CreateConstraintInEVMBlockPartition',
+        path: 'v1.CreateConstraintInEVMBlockPartition',
+      },
+    },
+    CrawlEvmBlock: {
+      key: 'CrawlEvmBlock',
+      path: 'v1.CrawlEvmBlock',
     },
   },
   V2: {
@@ -216,6 +228,10 @@ export const BULL_JOB_NAME = {
   REFRESH_ERC721_STATS: 'refresh:erc721-stats',
   SYNC_SOURCIFY: 'sync:sourcify',
   INSERT_VERIFY_BY_CODEHASH: 'job:insert-verify-by-codehash',
+  CRAWL_EVM_BLOCK: 'crawl:evm-block',
+  JOB_CREATE_EVM_BLOCK_PARTITION: 'job:create-evm-block-partition',
+  JOB_CHECK_EVM_BLOCK_CONSTRAINT: 'job:check-need-create-evm-block-constraint',
+  JOB_CREATE_EVM_BLOCK_CONSTRAINT: 'job:create-evm-block-constraint',
 };
 
 export const MSG_TYPE = {

src/services/evm/crawl_evm_block.service.ts

@@ -0,0 +1,119 @@
+import { Service } from '@ourparentcenter/moleculer-decorators-extended';
+import { PublicClient } from 'viem';
+import { BlockCheckpoint, EVMBlock } from '../../models';
+import EtherJsClient from '../../common/utils/etherjs_client';
+import BullableService, { QueueHandler } from '../../base/bullable.service';
+import { BULL_JOB_NAME, SERVICE } from './constant';
+import config from '../../../config.json' assert { type: 'json' };
+import knex from '../../common/utils/db_connection';
+import '../../../fetch-polyfill.js';
+
+@Service({
+  name: SERVICE.V1.CrawlEvmBlock.key,
+  version: 1,
+})
+export default class CrawlEvmBlockService extends BullableService {
+  viemJsClient!: PublicClient;
+
+  @QueueHandler({
+    queueName: BULL_JOB_NAME.CRAWL_EVM_BLOCK,
+    jobName: BULL_JOB_NAME.CRAWL_EVM_BLOCK,
+  })
+  async crawlEvmBlock() {
+    const latestBlock = parseInt(
+      (await this.viemJsClient.getBlockNumber()).toString(),
+      10
+    );
+    this.logger.info(`Latest block network: ${latestBlock}`);
+
+    let blockCheckpoint = await BlockCheckpoint.query().findOne({
+      job_name: BULL_JOB_NAME.CRAWL_EVM_BLOCK,
+    });
+    if (!blockCheckpoint) {
+      blockCheckpoint = await BlockCheckpoint.query().insert({
+        job_name: BULL_JOB_NAME.CRAWL_EVM_BLOCK,
+        height: config.crawlEvmBlock.startBlock,
+      });
+    }
+    this.logger.info(`Current block checkpoint: ${blockCheckpoint?.height}`);
+
+    // crawl block from startBlock to endBlock
+    const startBlock = blockCheckpoint.height + 1;
+    let endBlock = startBlock + config.crawlEvmBlock.blocksPerCall - 1;
+    if (endBlock > latestBlock) {
+      endBlock = latestBlock;
+    }
+    this.logger.info(`startBlock: ${startBlock} endBlock: ${endBlock}`);
+    const blockQueries = [];
+    for (let i = startBlock; i <= endBlock; i += 1) {
+      blockQueries.push(
+        this.viemJsClient.getBlock({
+          blockNumber: BigInt(i),
+          includeTransactions: true,
+        })
+      );
+    }
+    const res = await Promise.all(blockQueries);
+    // this.logger.info(res);
+    const EvmBlocks: EVMBlock[] = [];
+    if (res.length > 0) {
+      res.forEach((block) => {
+        if (block) {
+          EvmBlocks.push(
+            EVMBlock.fromJson({
+              height: block.number,
+              hash: block.hash,
+              parent_hash: block.parentHash,
+              date: new Date(parseInt(block.timestamp.toString(), 10) * 1000),
+              difficulty: block.difficulty,
+              gas_limit: block.gasLimit,
+              gas_used: block.gasUsed,
+              miner: block.miner,
+              nonce: block.nonce,
+              receipts_root: block.receiptsRoot,
+              state_root: block.stateRoot,
+              tx_count: block.transactions.length,
+              extra_data: block.extraData,
+              transactions: JSON.stringify(block.transactions, (key, value) =>
+                typeof value === 'bigint' ? value.toString() : value
+              ),
+            })
+          );
+        }
+      });
+      await knex.transaction(async (trx) => {
+        await EVMBlock.query().insert(EvmBlocks).transacting(trx);
+        await BlockCheckpoint.query()
+          .update(
+            BlockCheckpoint.fromJson({
+              job_name: BULL_JOB_NAME.CRAWL_EVM_BLOCK,
+              height: endBlock,
+            })
+          )
+          .where({
+            job_name: BULL_JOB_NAME.CRAWL_EVM_BLOCK,
+          })
+          .transacting(trx);
+      });
+    }
+  }
+
+  public async _start(): Promise<void> {
+    this.viemJsClient = EtherJsClient.getViemClient();
+    this.createJob(
+      BULL_JOB_NAME.CRAWL_EVM_BLOCK,
+      BULL_JOB_NAME.CRAWL_EVM_BLOCK,
+      {},
+      {
+        removeOnComplete: true,
+        removeOnFail: {
+          count: 3,
+        },
+        repeat: {
+          every: config.crawlEvmBlock.millisecondCrawl,
+        },
+      }
+    );
+    return super._start();
+  }
+}