From f39fc3835fa0cdd4a2b4e02314614682c4a8348e Mon Sep 17 00:00:00 2001 From: GuticaStefan Date: Fri, 22 Nov 2024 16:30:58 +0200 Subject: [PATCH 1/8] get real caller from ES for swap events --- .../providers/xexchange/xexchange.service.ts | 12 ++++++++++ .../src/services/indexer/indexer.service.ts | 22 +++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/libs/common/src/providers/xexchange/xexchange.service.ts b/libs/common/src/providers/xexchange/xexchange.service.ts index 470549a..0b24188 100644 --- a/libs/common/src/providers/xexchange/xexchange.service.ts +++ b/libs/common/src/providers/xexchange/xexchange.service.ts @@ -200,6 +200,18 @@ export class XExchangeService implements IProviderService { this.logger.error(`Unknown event topic ${event.topics[0]}. Event: ${JSON.stringify(event)}`); } } + + } + const swapEvents = events.filter(event => event instanceof XExchangeSwapEvent && (new Address(event.caller)).isContractAddress()); + const txHashes = swapEvents.map(event => event.txHash) + const txCallerPairs = await this.indexerService.getTxCallerPairs(txHashes) + for (const event of events) { + if (event instanceof XExchangeSwapEvent && (new Address(event.caller)).isContractAddress()) { + const matchingPair = txCallerPairs.find(pair => pair.tx === event.txHash); + if (matchingPair) { + event.caller = matchingPair.caller; + } + } } return events; diff --git a/libs/common/src/services/indexer/indexer.service.ts b/libs/common/src/services/indexer/indexer.service.ts index 80f8c62..7306c25 100644 --- a/libs/common/src/services/indexer/indexer.service.ts +++ b/libs/common/src/services/indexer/indexer.service.ts @@ -96,4 +96,26 @@ export class IndexerService { return logs; } + + @LogPerformanceAsync(MetricsEvents.SetIndexerDuration) + public async getTxCallerPairs(txHashes: string[]) { + const query = ElasticQuery.create() + .withPagination({ from: 0, size: 10000 }) + .withMustCondition([ + QueryType.Should(txHashes.map(txHash => new MatchQuery("_id", txHash))), + ]); + + const txCallerPairs: { + tx: any; caller: any + }[] = []; + + await this.elasticService.getScrollableList('operations', 'txHash', query, async (transactions) => { + const txCallerPairsTemp = transactions.map(transaction => ({ tx: transaction.txHash, caller: transaction.sender })); + txCallerPairs.push(...txCallerPairsTemp); + }, { + scrollTimeout: '10m', + }); + + return txCallerPairs; + } } From b490f96e5338a8ab122761ac58cd638c63b65346 Mon Sep 17 00:00:00 2001 From: GuticaStefan Date: Fri, 22 Nov 2024 17:05:19 +0200 Subject: [PATCH 2/8] improvements --- .../providers/xexchange/xexchange.service.ts | 24 ++++++++++++------- .../src/services/indexer/indexer.service.ts | 15 +++--------- 2 files changed, 19 insertions(+), 20 deletions(-) diff --git a/libs/common/src/providers/xexchange/xexchange.service.ts b/libs/common/src/providers/xexchange/xexchange.service.ts index 0b24188..8aeaa55 100644 --- a/libs/common/src/providers/xexchange/xexchange.service.ts +++ b/libs/common/src/providers/xexchange/xexchange.service.ts @@ -16,7 +16,7 @@ import { PairMetadata } from "../entities"; import { CacheService } from "@multiversx/sdk-nestjs-cache"; import BigNumber from "bignumber.js"; import { IndexerService } from "../../services/indexer"; -import { BinaryUtils, OriginLogger } from "@multiversx/sdk-nestjs-common"; +import { AddressUtils, BinaryUtils, OriginLogger } from "@multiversx/sdk-nestjs-common"; import { PAIR_EVENTS } from "@multiversx/sdk-exchange"; import { MultiversXApiService } from "../../services/multiversx.api"; import { PerformanceProfiler } from "@multiversx/sdk-nestjs-monitoring"; @@ -202,14 +202,22 @@ export class XExchangeService implements IProviderService { } } - const swapEvents = events.filter(event => event instanceof XExchangeSwapEvent && (new Address(event.caller)).isContractAddress()); - const txHashes = swapEvents.map(event => event.txHash) - const txCallerPairs = await this.indexerService.getTxCallerPairs(txHashes) + + const filteredEvents = events.filter(event => AddressUtils.isSmartContractAddress(event.caller)); + + const txHashes = filteredEvents.map(event => event.txHash) + + const transactions = await this.indexerService.getTxDetails(txHashes); + + const txToCallerMap = new Map( + transactions.map(transaction => [transaction.txHash, transaction.sender]) + ); + for (const event of events) { - if (event instanceof XExchangeSwapEvent && (new Address(event.caller)).isContractAddress()) { - const matchingPair = txCallerPairs.find(pair => pair.tx === event.txHash); - if (matchingPair) { - event.caller = matchingPair.caller; + if (AddressUtils.isSmartContractAddress(event.caller)) { + const callerFromMap = txToCallerMap.get(event.txHash); + if (callerFromMap) { + event.caller = callerFromMap; } } } diff --git a/libs/common/src/services/indexer/indexer.service.ts b/libs/common/src/services/indexer/indexer.service.ts index 7306c25..46b2c05 100644 --- a/libs/common/src/services/indexer/indexer.service.ts +++ b/libs/common/src/services/indexer/indexer.service.ts @@ -98,24 +98,15 @@ export class IndexerService { } @LogPerformanceAsync(MetricsEvents.SetIndexerDuration) - public async getTxCallerPairs(txHashes: string[]) { + public async getTxDetails(txHashes: string[]) { const query = ElasticQuery.create() .withPagination({ from: 0, size: 10000 }) .withMustCondition([ QueryType.Should(txHashes.map(txHash => new MatchQuery("_id", txHash))), ]); - const txCallerPairs: { - tx: any; caller: any - }[] = []; + const transactions = await this.elasticService.getList('operations', 'txHash', query); - await this.elasticService.getScrollableList('operations', 'txHash', query, async (transactions) => { - const txCallerPairsTemp = transactions.map(transaction => ({ tx: transaction.txHash, caller: transaction.sender })); - txCallerPairs.push(...txCallerPairsTemp); - }, { - scrollTimeout: '10m', - }); - - return txCallerPairs; + return transactions; } } From f8789d5a3aeb07abf895ac1e017060321d8e9684 Mon Sep 17 00:00:00 2001 From: GuticaStefan Date: Fri, 22 Nov 2024 17:37:19 +0200 Subject: [PATCH 3/8] update event caller address for all events --- .../data-integration.service.ts | 26 ++++++++++++++++++- .../providers/xexchange/xexchange.service.ts | 21 +-------------- 2 files changed, 26 insertions(+), 21 deletions(-) diff --git a/apps/api/src/endpoints/data-integration/data-integration.service.ts b/apps/api/src/endpoints/data-integration/data-integration.service.ts index 2c0c63b..408ddee 100644 --- a/apps/api/src/endpoints/data-integration/data-integration.service.ts +++ b/apps/api/src/endpoints/data-integration/data-integration.service.ts @@ -4,7 +4,7 @@ import { IndexerService, JoinExitEvent, LatestBlockResponse, MultiversXApiService, PairResponse, SwapEvent, XExchangeService, } from "@mvx-monorepo/common"; -import { OriginLogger } from "@multiversx/sdk-nestjs-common"; +import { AddressUtils, OriginLogger } from "@multiversx/sdk-nestjs-common"; import { IProviderService } from "@mvx-monorepo/common/providers/interface"; import { GeneralEvent } from "@mvx-monorepo/common/providers/entities/general.event"; import { OneDexService } from "@mvx-monorepo/common/providers"; @@ -83,6 +83,8 @@ export class DataIntegrationService { allEvents.push(...event); } + await this.updateEventsCaller(allEvents); + const sortedEvents = allEvents.sort((a, b) => { if (a.block.blockTimestamp !== b.block.blockTimestamp) { return a.block.blockTimestamp - b.block.blockTimestamp; @@ -136,4 +138,26 @@ export class DataIntegrationService { } return events; } + + private async updateEventsCaller(events: ({ block: Block } & (SwapEvent | JoinExitEvent))[]) { + const filteredEvents = events.filter(event => AddressUtils.isSmartContractAddress(event.maker)); + + const txHashes = filteredEvents.map(event => event.txnId) + + const transactions = await this.indexerService.getTxDetails(txHashes); + + const txToCallerMap = new Map( + transactions.map(transaction => [transaction.txHash, transaction.sender]) + ); + + for (const event of events) { + if (AddressUtils.isSmartContractAddress(event.maker)) { + const callerFromMap = txToCallerMap.get(event.txnId); + if (callerFromMap) { + event.maker = callerFromMap; + } + } + } + console.log(txToCallerMap) + } } diff --git a/libs/common/src/providers/xexchange/xexchange.service.ts b/libs/common/src/providers/xexchange/xexchange.service.ts index 8aeaa55..d440cb5 100644 --- a/libs/common/src/providers/xexchange/xexchange.service.ts +++ b/libs/common/src/providers/xexchange/xexchange.service.ts @@ -16,7 +16,7 @@ import { PairMetadata } from "../entities"; import { CacheService } from "@multiversx/sdk-nestjs-cache"; import BigNumber from "bignumber.js"; import { IndexerService } from "../../services/indexer"; -import { AddressUtils, BinaryUtils, OriginLogger } from "@multiversx/sdk-nestjs-common"; +import { BinaryUtils, OriginLogger } from "@multiversx/sdk-nestjs-common"; import { PAIR_EVENTS } from "@multiversx/sdk-exchange"; import { MultiversXApiService } from "../../services/multiversx.api"; import { PerformanceProfiler } from "@multiversx/sdk-nestjs-monitoring"; @@ -203,25 +203,6 @@ export class XExchangeService implements IProviderService { } - const filteredEvents = events.filter(event => AddressUtils.isSmartContractAddress(event.caller)); - - const txHashes = filteredEvents.map(event => event.txHash) - - const transactions = await this.indexerService.getTxDetails(txHashes); - - const txToCallerMap = new Map( - transactions.map(transaction => [transaction.txHash, transaction.sender]) - ); - - for (const event of events) { - if (AddressUtils.isSmartContractAddress(event.caller)) { - const callerFromMap = txToCallerMap.get(event.txHash); - if (callerFromMap) { - event.caller = callerFromMap; - } - } - } - return events; } From 36b9861820767b903033b9a786c366618d57bdb7 Mon Sep 17 00:00:00 2001 From: GuticaStefan Date: Mon, 25 Nov 2024 09:29:16 +0200 Subject: [PATCH 4/8] remove console log --- .../src/endpoints/data-integration/data-integration.service.ts | 3 +-- libs/common/src/providers/xexchange/xexchange.service.ts | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/apps/api/src/endpoints/data-integration/data-integration.service.ts b/apps/api/src/endpoints/data-integration/data-integration.service.ts index 408ddee..fc2b4fc 100644 --- a/apps/api/src/endpoints/data-integration/data-integration.service.ts +++ b/apps/api/src/endpoints/data-integration/data-integration.service.ts @@ -142,7 +142,7 @@ export class DataIntegrationService { private async updateEventsCaller(events: ({ block: Block } & (SwapEvent | JoinExitEvent))[]) { const filteredEvents = events.filter(event => AddressUtils.isSmartContractAddress(event.maker)); - const txHashes = filteredEvents.map(event => event.txnId) + const txHashes = filteredEvents.map(event => event.txnId); const transactions = await this.indexerService.getTxDetails(txHashes); @@ -158,6 +158,5 @@ export class DataIntegrationService { } } } - console.log(txToCallerMap) } } diff --git a/libs/common/src/providers/xexchange/xexchange.service.ts b/libs/common/src/providers/xexchange/xexchange.service.ts index d440cb5..470549a 100644 --- a/libs/common/src/providers/xexchange/xexchange.service.ts +++ b/libs/common/src/providers/xexchange/xexchange.service.ts @@ -200,7 +200,6 @@ export class XExchangeService implements IProviderService { this.logger.error(`Unknown event topic ${event.topics[0]}. Event: ${JSON.stringify(event)}`); } } - } return events; From bc0a8e61be5953ed767382577534f3fa7e1d058e Mon Sep 17 00:00:00 2001 From: GuticaStefan Date: Mon, 25 Nov 2024 14:13:16 +0200 Subject: [PATCH 5/8] add await for async functions --- libs/common/src/providers/onedex/onedex.service.ts | 2 +- libs/common/src/providers/xexchange/xexchange.service.ts | 4 ++-- .../src/services/multiversx.api/multiversx.api.service.ts | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/libs/common/src/providers/onedex/onedex.service.ts b/libs/common/src/providers/onedex/onedex.service.ts index cd890ef..3042415 100644 --- a/libs/common/src/providers/onedex/onedex.service.ts +++ b/libs/common/src/providers/onedex/onedex.service.ts @@ -109,7 +109,7 @@ export class OneDexService implements IProviderService { public async getPairsMetadata(): Promise { return await this.cacheService.getOrSet( CacheInfo.OneDexPairsMetadata().key, - () => this.getPairsMetadataRaw(), + async () => await this.getPairsMetadataRaw(), CacheInfo.OneDexPairsMetadata().ttl, ); } diff --git a/libs/common/src/providers/xexchange/xexchange.service.ts b/libs/common/src/providers/xexchange/xexchange.service.ts index 470549a..bcda982 100644 --- a/libs/common/src/providers/xexchange/xexchange.service.ts +++ b/libs/common/src/providers/xexchange/xexchange.service.ts @@ -80,7 +80,7 @@ export class XExchangeService implements IProviderService { public async getPairsMetadata(): Promise { return await this.cacheService.getOrSet( CacheInfo.PairsMetadata().key, - () => this.getPairsMetadataRaw(), + async () => await this.getPairsMetadataRaw(), CacheInfo.PairsMetadata().ttl, ); } @@ -115,7 +115,7 @@ export class XExchangeService implements IProviderService { public async getPairFeePercent(pairAddress: string): Promise { return await this.cacheService.getOrSet( CacheInfo.PairFeePercent(pairAddress).key, - () => this.getPairFeePercentRaw(pairAddress), + async () => await this.getPairFeePercentRaw(pairAddress), CacheInfo.PairFeePercent(pairAddress).ttl, ); } diff --git a/libs/common/src/services/multiversx.api/multiversx.api.service.ts b/libs/common/src/services/multiversx.api/multiversx.api.service.ts index dc27f5c..e91c59f 100644 --- a/libs/common/src/services/multiversx.api/multiversx.api.service.ts +++ b/libs/common/src/services/multiversx.api/multiversx.api.service.ts @@ -18,7 +18,7 @@ export class MultiversXApiService { public async getToken(identifier: string): Promise { return await this.cacheService.getOrSet( CacheInfo.Token(identifier).key, - () => this.getTokenRaw(identifier), + async () => await this.getTokenRaw(identifier), CacheInfo.Token(identifier).ttl, ); } From f5199305b9c108307ea0b4b3d2be45856c82875b Mon Sep 17 00:00:00 2001 From: GuticaStefan Date: Mon, 25 Nov 2024 15:12:53 +0200 Subject: [PATCH 6/8] fetch txDetails in batches --- .../data-integration.service.ts | 17 ++++++++++++++- .../src/services/indexer/indexer.service.ts | 21 ++++++++++++------- .../multiversx.api/multiversx.api.service.ts | 2 +- 3 files changed, 31 insertions(+), 9 deletions(-) diff --git a/apps/api/src/endpoints/data-integration/data-integration.service.ts b/apps/api/src/endpoints/data-integration/data-integration.service.ts index fc2b4fc..4689cd3 100644 --- a/apps/api/src/endpoints/data-integration/data-integration.service.ts +++ b/apps/api/src/endpoints/data-integration/data-integration.service.ts @@ -144,7 +144,7 @@ export class DataIntegrationService { const txHashes = filteredEvents.map(event => event.txnId); - const transactions = await this.indexerService.getTxDetails(txHashes); + const transactions = await this.getTxDetailsInBatches(txHashes, 200); const txToCallerMap = new Map( transactions.map(transaction => [transaction.txHash, transaction.sender]) @@ -159,4 +159,19 @@ export class DataIntegrationService { } } } + + private async getTxDetailsInBatches(txHashes: string[], batchSize: number) { + let transactions: any[] = []; + let start = 0; + while (start < txHashes.length) { + const txHashesBatch = txHashes.slice(start, start + batchSize); + + const transactionsBatch = await this.indexerService.getTxDetails(txHashesBatch); + + transactions.push(...transactionsBatch); + start += batchSize; + } + + return transactions; + } } diff --git a/libs/common/src/services/indexer/indexer.service.ts b/libs/common/src/services/indexer/indexer.service.ts index 46b2c05..eb7fba3 100644 --- a/libs/common/src/services/indexer/indexer.service.ts +++ b/libs/common/src/services/indexer/indexer.service.ts @@ -99,14 +99,21 @@ export class IndexerService { @LogPerformanceAsync(MetricsEvents.SetIndexerDuration) public async getTxDetails(txHashes: string[]) { - const query = ElasticQuery.create() - .withPagination({ from: 0, size: 10000 }) - .withMustCondition([ - QueryType.Should(txHashes.map(txHash => new MatchQuery("_id", txHash))), - ]); + try { + const query = ElasticQuery.create() + .withPagination({ from: 0, size: 10000 }) + .withMustCondition([ + QueryType.Should(txHashes.map(txHash => new MatchQuery("_id", txHash))), + ]); - const transactions = await this.elasticService.getList('operations', 'txHash', query); + const transactions = await this.elasticService.getList('operations', 'txHash', query); - return transactions; + return transactions; + } catch (error) { + this.logger.error(`Failed to get txDetails`); + this.logger.error(error); + + return []; + } } } diff --git a/libs/common/src/services/multiversx.api/multiversx.api.service.ts b/libs/common/src/services/multiversx.api/multiversx.api.service.ts index e91c59f..0e12eb4 100644 --- a/libs/common/src/services/multiversx.api/multiversx.api.service.ts +++ b/libs/common/src/services/multiversx.api/multiversx.api.service.ts @@ -42,7 +42,7 @@ export class MultiversXApiService { public async getContractDeployInfo(address: string): Promise<{ deployTxHash?: string, deployedAt?: number }> { return await this.cacheService.getOrSet( CacheInfo.ContractDeployInfo(address).key, - () => this.getContractDeployInfoRaw(address), + async () => await this.getContractDeployInfoRaw(address), CacheInfo.ContractDeployInfo(address).ttl, ); } From ca8963a52005829b6eb39006e2157c7503843716 Mon Sep 17 00:00:00 2001 From: GuticaStefan Date: Mon, 25 Nov 2024 15:14:55 +0200 Subject: [PATCH 7/8] use const for never reassigned variable --- .../src/endpoints/data-integration/data-integration.service.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/api/src/endpoints/data-integration/data-integration.service.ts b/apps/api/src/endpoints/data-integration/data-integration.service.ts index 4689cd3..8435e4f 100644 --- a/apps/api/src/endpoints/data-integration/data-integration.service.ts +++ b/apps/api/src/endpoints/data-integration/data-integration.service.ts @@ -161,7 +161,7 @@ export class DataIntegrationService { } private async getTxDetailsInBatches(txHashes: string[], batchSize: number) { - let transactions: any[] = []; + const transactions: any[] = []; let start = 0; while (start < txHashes.length) { const txHashesBatch = txHashes.slice(start, start + batchSize); From c544da436d49fea95d6c72adf51be87b8940f07a Mon Sep 17 00:00:00 2001 From: GuticaStefan Date: Mon, 25 Nov 2024 15:42:29 +0200 Subject: [PATCH 8/8] use BatchUtils to split array into batches --- .../data-integration/data-integration.service.ts | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/apps/api/src/endpoints/data-integration/data-integration.service.ts b/apps/api/src/endpoints/data-integration/data-integration.service.ts index 8435e4f..4d1a690 100644 --- a/apps/api/src/endpoints/data-integration/data-integration.service.ts +++ b/apps/api/src/endpoints/data-integration/data-integration.service.ts @@ -4,7 +4,7 @@ import { IndexerService, JoinExitEvent, LatestBlockResponse, MultiversXApiService, PairResponse, SwapEvent, XExchangeService, } from "@mvx-monorepo/common"; -import { AddressUtils, OriginLogger } from "@multiversx/sdk-nestjs-common"; +import { AddressUtils, BatchUtils, OriginLogger } from "@multiversx/sdk-nestjs-common"; import { IProviderService } from "@mvx-monorepo/common/providers/interface"; import { GeneralEvent } from "@mvx-monorepo/common/providers/entities/general.event"; import { OneDexService } from "@mvx-monorepo/common/providers"; @@ -162,14 +162,12 @@ export class DataIntegrationService { private async getTxDetailsInBatches(txHashes: string[], batchSize: number) { const transactions: any[] = []; - let start = 0; - while (start < txHashes.length) { - const txHashesBatch = txHashes.slice(start, start + batchSize); + const txHashesBatches = BatchUtils.splitArrayIntoChunks(txHashes, batchSize); + for (const txHashesBatch of txHashesBatches) { const transactionsBatch = await this.indexerService.getTxDetails(txHashesBatch); transactions.push(...transactionsBatch); - start += batchSize; } return transactions;