From 813e74476d628f24c6af64a6ee4a2b4cb4c1948c Mon Sep 17 00:00:00 2001 From: phamphong9981 Date: Fri, 3 May 2024 13:57:17 +0700 Subject: [PATCH 1/4] feat: erc20 statistic --- .../evm/20240502080101_erc20_statistic.ts | 16 ++ src/models/erc20_contract.ts | 11 ++ src/models/erc20_statistic.ts | 44 ++++++ src/models/index.ts | 1 + src/services/evm/erc20.service.ts | 142 ++++++++++++------ test/unit/services/erc20/erc20.spec.ts | 95 +++++++++++- 6 files changed, 264 insertions(+), 45 deletions(-) create mode 100644 migrations/evm/20240502080101_erc20_statistic.ts create mode 100644 src/models/erc20_statistic.ts diff --git a/migrations/evm/20240502080101_erc20_statistic.ts b/migrations/evm/20240502080101_erc20_statistic.ts new file mode 100644 index 000000000..8ec81e57e --- /dev/null +++ b/migrations/evm/20240502080101_erc20_statistic.ts @@ -0,0 +1,16 @@ +import { Knex } from 'knex'; + +export async function up(knex: Knex): Promise { + await knex.schema.createTable('erc20_statistic', (table) => { + table.increments(); + table.integer('erc20_contract_id').notNullable(); + table.foreign('erc20_contract_id').references('erc20_contract.id'); + table.integer('total_holder'); + table.date('date').index().notNullable(); + table.unique(['erc20_contract_id', 'date']); + }); +} + +export async function down(knex: Knex): Promise { + await knex.schema.dropTableIfExists('erc20_statistic'); +} diff --git a/src/models/erc20_contract.ts b/src/models/erc20_contract.ts index 7694b430c..51a58a3c1 100644 --- a/src/models/erc20_contract.ts +++ b/src/models/erc20_contract.ts @@ -3,8 +3,11 @@ import BaseModel from './base'; import { EVMSmartContract } from './evm_smart_contract'; // eslint-disable-next-line import/no-cycle import { Erc20Activity } from './erc20_activity'; +import { AccountBalance } from './account_balance'; export class Erc20Contract extends BaseModel { + [relation: string]: any; + static softDelete = false; id!: number; @@ -61,6 +64,14 @@ export class Erc20Contract extends BaseModel { from: 'erc20_contract.address', }, }, + holders: { + relation: Model.HasManyRelation, + modelClass: AccountBalance, + join: { + to: 'account_balance.denom', + from: 'erc20_contract.address', + }, + }, }; } diff --git a/src/models/erc20_statistic.ts b/src/models/erc20_statistic.ts new file mode 100644 index 000000000..2dfadfa7a --- /dev/null +++ b/src/models/erc20_statistic.ts @@ -0,0 +1,44 @@ +import { Model } from 'objection'; +import BaseModel from './base'; +import { Erc20Contract } from './erc20_contract'; + +export class Erc20Statistic extends BaseModel { + static softDelete = false; + + [relation: string]: any; + + date!: Date; + + erc20_contract_id!: number; + + total_holder!: number; + + static get tableName() { + return 'erc20_statistic'; + } + + static get jsonSchema() { + return { + type: 'object', + required: ['erc20_contract_id', 'total_holder', 'date'], + properties: { + erc20_contract_id: { type: 'number' }, + total_holder: { type: 'number' }, + date: { type: 'object' }, + }, + }; + } + + static get relationMappings() { + return { + erc20_contract: { + relation: Model.BelongsToOneRelation, + modelClass: Erc20Contract, + join: { + from: 'erc20_statistic.erc20_contract_id', + to: 'erc20_contract.id', + }, + }, + }; + } +} diff --git a/src/models/index.ts b/src/models/index.ts index f4ce4cfae..a7f5f75c1 100644 --- a/src/models/index.ts +++ b/src/models/index.ts @@ -43,3 +43,4 @@ export * from './evm_internal_transaction'; export * from './erc721_activity'; export * from './erc721_token'; export * from './account_balance'; +export * from './erc20_statistic'; diff --git a/src/services/evm/erc20.service.ts b/src/services/evm/erc20.service.ts index f6bb93226..e09afa7dc 100644 --- a/src/services/evm/erc20.service.ts +++ b/src/services/evm/erc20.service.ts @@ -9,10 +9,16 @@ import { PublicClient, getContract } from 'viem'; import config from '../../../config.json' assert { type: 'json' }; import '../../../fetch-polyfill.js'; import BullableService, { QueueHandler } from '../../base/bullable.service'; -import { SERVICE as COSMOS_SERVICE } from '../../common'; +import { SERVICE as COSMOS_SERVICE, Config } from '../../common'; import knex from '../../common/utils/db_connection'; import EtherJsClient from '../../common/utils/etherjs_client'; -import { BlockCheckpoint, EVMSmartContract, EvmEvent } from '../../models'; +import { + Block, + BlockCheckpoint, + EVMSmartContract, + Erc20Statistic, + EvmEvent, +} from '../../models'; import { AccountBalance } from '../../models/account_balance'; import { Erc20Activity } from '../../models/erc20_activity'; import { Erc20Contract } from '../../models/erc20_contract'; @@ -20,6 +26,7 @@ import { BULL_JOB_NAME, SERVICE as EVM_SERVICE, SERVICE } from './constant'; import { ERC20_EVENT_TOPIC0, Erc20Handler } from './erc20_handler'; import { convertEthAddressToBech32Address } from './utils'; +const { NODE_ENV } = Config; @Service({ name: EVM_SERVICE.V1.Erc20.key, version: 1, @@ -154,6 +161,7 @@ export default class Erc20Service extends BullableService { [BULL_JOB_NAME.HANDLE_ERC20_ACTIVITY], config.erc20.key ); + await this.handleStatistic(startBlock); // get Erc20 activities let erc20Activities = await this.getErc20Activities(startBlock, endBlock); // get missing Account @@ -370,50 +378,96 @@ export default class Erc20Service extends BullableService { })); } + async handleStatistic(startBlock: number) { + const systemDate = ( + await Block.query() + .where('height', startBlock + 1) + .first() + .throwIfNotFound() + ).time; + const lastUpdatedDate = (await Erc20Statistic.query().max('date').first()) + ?.max; + if (lastUpdatedDate) { + systemDate.setHours(0, 0, 0, 0); + lastUpdatedDate.setHours(0, 0, 0, 0); + if (systemDate > lastUpdatedDate) { + await this.handleTotalHolderStatistic(systemDate); + } + } else { + await this.handleTotalHolderStatistic(systemDate); + } + } + + async handleTotalHolderStatistic(systemDate: Date) { + const totalHolder = await Erc20Contract.query() + .joinRelated('holders') + .where('erc20_contract.track', true) + .groupBy('erc20_contract.id') + .select( + 'erc20_contract.id as erc20_contract_id', + knex.raw( + 'count(CASE when holders.amount > 0 THEN 1 ELSE null END) as count' + ) + ); + if (totalHolder.length > 0) { + await Erc20Statistic.query().insert( + totalHolder.map((e) => + Erc20Statistic.fromJson({ + erc20_contract_id: e.erc20_contract_id, + total_holder: e.count, + date: systemDate, + }) + ) + ); + } + } + public async _start(): Promise { this.viemClient = EtherJsClient.getViemClient(); - await this.createJob( - BULL_JOB_NAME.HANDLE_ERC20_CONTRACT, - BULL_JOB_NAME.HANDLE_ERC20_CONTRACT, - {}, - { - removeOnComplete: true, - removeOnFail: { - count: 3, - }, - repeat: { - every: config.erc20.millisecondRepeatJob, - }, - } - ); - await this.createJob( - BULL_JOB_NAME.HANDLE_ERC20_ACTIVITY, - BULL_JOB_NAME.HANDLE_ERC20_ACTIVITY, - {}, - { - removeOnComplete: true, - removeOnFail: { - count: 3, - }, - repeat: { - every: config.erc20.millisecondRepeatJob, - }, - } - ); - await this.createJob( - BULL_JOB_NAME.HANDLE_ERC20_BALANCE, - BULL_JOB_NAME.HANDLE_ERC20_BALANCE, - {}, - { - removeOnComplete: true, - removeOnFail: { - count: 3, - }, - repeat: { - every: config.erc20.millisecondRepeatJob, - }, - } - ); + if (NODE_ENV !== 'test') { + await this.createJob( + BULL_JOB_NAME.HANDLE_ERC20_CONTRACT, + BULL_JOB_NAME.HANDLE_ERC20_CONTRACT, + {}, + { + removeOnComplete: true, + removeOnFail: { + count: 3, + }, + repeat: { + every: config.erc20.millisecondRepeatJob, + }, + } + ); + await this.createJob( + BULL_JOB_NAME.HANDLE_ERC20_ACTIVITY, + BULL_JOB_NAME.HANDLE_ERC20_ACTIVITY, + {}, + { + removeOnComplete: true, + removeOnFail: { + count: 3, + }, + repeat: { + every: config.erc20.millisecondRepeatJob, + }, + } + ); + await this.createJob( + BULL_JOB_NAME.HANDLE_ERC20_BALANCE, + BULL_JOB_NAME.HANDLE_ERC20_BALANCE, + {}, + { + removeOnComplete: true, + removeOnFail: { + count: 3, + }, + repeat: { + every: config.erc20.millisecondRepeatJob, + }, + } + ); + } return super._start(); } } diff --git a/test/unit/services/erc20/erc20.spec.ts b/test/unit/services/erc20/erc20.spec.ts index 9cb0eef9f..465021c1f 100644 --- a/test/unit/services/erc20/erc20.spec.ts +++ b/test/unit/services/erc20/erc20.spec.ts @@ -1,5 +1,12 @@ -import { AfterAll, BeforeAll, Describe, Test } from '@jest-decorated/core'; +import { + AfterAll, + BeforeAll, + BeforeEach, + Describe, + Test, +} from '@jest-decorated/core'; import { ServiceBroker } from 'moleculer'; +import _ from 'lodash'; import knex from '../../../../src/common/utils/db_connection'; import { Account, @@ -7,6 +14,7 @@ import { EVMTransaction, Erc20Activity, Erc20Contract, + Erc20Statistic, EvmEvent, } from '../../../../src/models'; import Erc20Service from '../../../../src/services/evm/erc20.service'; @@ -60,6 +68,7 @@ export default class Erc20Test { @BeforeAll() async initSuite() { + this.erc20Service.getQueueManager().stopAll(); await this.broker.start(); await knex.raw( 'TRUNCATE TABLE erc20_contract, account, erc20_activity, evm_smart_contract, evm_event, evm_transaction RESTART IDENTITY CASCADE' @@ -77,6 +86,13 @@ export default class Erc20Test { await this.broker.stop(); } + @BeforeEach() + async initSuiteBeforeEach() { + await knex.raw( + 'TRUNCATE TABLE erc20_activity, account, erc20_statistic RESTART IDENTITY CASCADE' + ); + } + @Test('test getErc20Activities') async testGetErc20Activities() { const fromAccount = Account.fromJson({ @@ -189,4 +205,81 @@ export default class Erc20Test { expect(result[1].from_account_id).toEqual(fromAccount.id); expect(result[1].to_account_id).toEqual(toAccount.id); } + + @Test('test handleTotalHolderStatistic') + public async testHandleTotalHolderStatistic() { + const accounts = [ + Account.fromJson({ + id: 345, + address: 'xczfsdfsfsdg', + balances: [], + spendable_balances: [], + type: '', + pubkey: '', + account_number: 2, + sequence: 432, + evm_address: '0xghgfhfghfg', + account_balances: [ + { + denom: this.evmSmartContract.address, + amount: 123, + }, + { + denom: this.evmSmartContract2.address, + amount: 1234, + }, + ], + }), + Account.fromJson({ + id: 456, + address: 'cbbvb', + balances: [], + spendable_balances: [], + type: '', + pubkey: '', + account_number: 2, + sequence: 432, + evm_address: '0xhgfhfghgfg', + account_balances: [ + { + denom: this.evmSmartContract.address, + amount: 0, + }, + { + denom: this.evmSmartContract2.address, + amount: -1234, + }, + ], + }), + Account.fromJson({ + id: 567, + address: 'xzxzcvv ', + balances: [], + spendable_balances: [], + type: '', + pubkey: '', + account_number: 2, + sequence: 432, + evm_address: '0xdgfsdgs4', + account_balances: [ + { + denom: this.evmSmartContract.address, + amount: 1, + }, + ], + }), + ]; + await Account.query().insertGraph(accounts); + const date = new Date('2023-01-12T00:53:57.000Z'); + await this.erc20Service.handleTotalHolderStatistic(date); + const erc20Statistics = _.keyBy( + await Erc20Statistic.query(), + 'erc20_contract_id' + ); + expect(erc20Statistics[444]).toMatchObject({ + total_holder: 2, + }); + // because of track false + expect(erc20Statistics[445]).toBeUndefined(); + } } From 84f2d3376d03dcb86b526c7b2cd9230b7e07f0e9 Mon Sep 17 00:00:00 2001 From: phamphong9981 Date: Wed, 15 May 2024 09:48:52 +0700 Subject: [PATCH 2/4] refactor: review --- src/services/cw20/cw20.service.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/services/cw20/cw20.service.ts b/src/services/cw20/cw20.service.ts index 33e8fb962..d79eb64ba 100644 --- a/src/services/cw20/cw20.service.ts +++ b/src/services/cw20/cw20.service.ts @@ -272,7 +272,7 @@ export default class Cw20Service extends BullableService { } async handleTotalHolderStatistic(systemDate: Date) { - const totalHolder = await Cw20Contract.query() + const totalHolders = await Cw20Contract.query() .joinRelated('holders') .where('cw20_contract.track', true) .groupBy('cw20_contract.id') @@ -282,9 +282,9 @@ export default class Cw20Service extends BullableService { 'count(CASE when holders.amount > 0 THEN 1 ELSE null END) as count' ) ); - if (totalHolder.length > 0) { + if (totalHolders.length > 0) { await CW20TotalHolderStats.query().insert( - totalHolder.map((e) => + totalHolders.map((e) => CW20TotalHolderStats.fromJson({ cw20_contract_id: e.cw20_contract_id, total_holder: e.count, From 3dc6694a6e982a3bcad3b299555e35acbe53c10c Mon Sep 17 00:00:00 2001 From: phamphong9981 Date: Wed, 15 May 2024 16:08:32 +0700 Subject: [PATCH 3/4] refactor: review --- src/services/cw20/cw20.service.ts | 6 +++--- src/services/evm/erc20.service.ts | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/services/cw20/cw20.service.ts b/src/services/cw20/cw20.service.ts index d79eb64ba..33e8fb962 100644 --- a/src/services/cw20/cw20.service.ts +++ b/src/services/cw20/cw20.service.ts @@ -272,7 +272,7 @@ export default class Cw20Service extends BullableService { } async handleTotalHolderStatistic(systemDate: Date) { - const totalHolders = await Cw20Contract.query() + const totalHolder = await Cw20Contract.query() .joinRelated('holders') .where('cw20_contract.track', true) .groupBy('cw20_contract.id') @@ -282,9 +282,9 @@ export default class Cw20Service extends BullableService { 'count(CASE when holders.amount > 0 THEN 1 ELSE null END) as count' ) ); - if (totalHolders.length > 0) { + if (totalHolder.length > 0) { await CW20TotalHolderStats.query().insert( - totalHolders.map((e) => + totalHolder.map((e) => CW20TotalHolderStats.fromJson({ cw20_contract_id: e.cw20_contract_id, total_holder: e.count, diff --git a/src/services/evm/erc20.service.ts b/src/services/evm/erc20.service.ts index 9ab06792d..38004b549 100644 --- a/src/services/evm/erc20.service.ts +++ b/src/services/evm/erc20.service.ts @@ -402,7 +402,7 @@ export default class Erc20Service extends BullableService { } async handleTotalHolderStatistic(systemDate: Date) { - const totalHolder = await Erc20Contract.query() + const totalHolders = await Erc20Contract.query() .joinRelated('holders') .where('erc20_contract.track', true) .groupBy('erc20_contract.id') @@ -412,9 +412,9 @@ export default class Erc20Service extends BullableService { 'count(CASE when holders.amount > 0 THEN 1 ELSE null END) as count' ) ); - if (totalHolder.length > 0) { + if (totalHolders.length > 0) { await Erc20Statistic.query().insert( - totalHolder.map((e) => + totalHolders.map((e) => Erc20Statistic.fromJson({ erc20_contract_id: e.erc20_contract_id, total_holder: e.count, From 70f233574b912883880ca37818ccd2cba6607d0b Mon Sep 17 00:00:00 2001 From: phamphong9981 Date: Wed, 3 Jul 2024 10:57:23 +0700 Subject: [PATCH 4/4] refactor: review --- src/services/evm/erc20.service.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/services/evm/erc20.service.ts b/src/services/evm/erc20.service.ts index 25a0194eb..ab167f404 100644 --- a/src/services/evm/erc20.service.ts +++ b/src/services/evm/erc20.service.ts @@ -387,8 +387,8 @@ export default class Erc20Service extends BullableService { .first() .throwIfNotFound() ).time; - const lastUpdatedDate = (await Erc20Statistic.query().max('date').first()) - ?.max; + const lastUpdatedRecord = await Erc20Statistic.query().max('date').first(); + const lastUpdatedDate = lastUpdatedRecord?.max; if (lastUpdatedDate) { systemDate.setHours(0, 0, 0, 0); lastUpdatedDate.setHours(0, 0, 0, 0);