From ef12e8e1c368fe7ae2792e64ff8cf8c80723f68c Mon Sep 17 00:00:00 2001 From: djordjekovac Date: Fri, 12 Apr 2024 11:18:57 +0200 Subject: [PATCH 01/13] Support staking for Neuro (#3124) * Added get latest service agreement data command, removed listening on asset created grouped events * Updated default hash function id, prioritizing commits and proofs for assets with new score function id * Added permanent command * Fixes after testing for latest-service-agreement command * Updated version of contracts that are deployed for hardhat 1 * Added support for neuro * comment removed, logging improved * Default score function id for neuro updated --------- Co-authored-by: Mihajlo Pavlovic --- ...blockchain-get-latest-service-agreement.js | 186 ++++++++++++++++++ .../get-latest-service-agreement.js | 71 +++++++ .../blockchain-epoch-check-command.js | 5 +- src/constants/constants.js | 18 +- .../bid-suggestion-http-api-controller-v0.js | 9 +- .../implementation/hardhat/hardhat-service.js | 3 - .../ot-parachain/ot-parachain-service.js | 4 - .../service-agreement-repository.js | 18 +- .../repository/repository-module-manager.js | 6 + .../blockchain-event-listener-service.js | 87 -------- .../setup-macos-environment.sh | 2 +- 11 files changed, 289 insertions(+), 120 deletions(-) create mode 100644 src/commands/common/get-latest-service-agreement/blockchain-get-latest-service-agreement.js create mode 100644 src/commands/common/get-latest-service-agreement/get-latest-service-agreement.js diff --git a/src/commands/common/get-latest-service-agreement/blockchain-get-latest-service-agreement.js b/src/commands/common/get-latest-service-agreement/blockchain-get-latest-service-agreement.js new file mode 100644 index 000000000..a2085b706 --- /dev/null +++ b/src/commands/common/get-latest-service-agreement/blockchain-get-latest-service-agreement.js @@ -0,0 +1,186 @@ +import Command from '../../command.js'; +import { + CONTENT_ASSET_HASH_FUNCTION_ID, + EXPECTED_TRANSACTION_ERRORS, + GET_LATEST_SERVICE_AGREEMENT_FREQUENCY_MILLS, + SERVICE_AGREEMENT_SOURCES, +} from '../../../constants/constants.js'; + +const BATCH_SIZE = 50; + +class BlockchainGetLatestServiceAgreement extends Command { + constructor(ctx) { + super(ctx); + this.repositoryModuleManager = ctx.repositoryModuleManager; + this.blockchainModuleManager = ctx.blockchainModuleManager; + this.serviceAgreementService = ctx.serviceAgreementService; + this.ualService = ctx.ualService; + } + + /** + * Executes command and produces one or more events + * @param command + */ + async execute(command) { + const { blockchain } = command.data; + + const assetStorageContractAddresses = + this.blockchainModuleManager.getAssetStorageContractAddresses(blockchain); + + await Promise.all( + assetStorageContractAddresses.map((contract) => + this.updateAgreementDataForAssetContract(contract, blockchain), + ), + ); + + return Command.repeat(); + } + + async updateAgreementDataForAssetContract(contract, blockchain) { + this.logger.info( + `Get latest service agreement: Starting get latest service agreement command for blockchain: ${blockchain}`, + ); + let latestBlockchainTokenId; + try { + latestBlockchainTokenId = await this.blockchainModuleManager.getLatestTokenId( + blockchain, + contract, + ); + } catch (error) { + if (error.message.includes(EXPECTED_TRANSACTION_ERRORS.NO_MINTED_ASSETS)) { + this.logger.info( + `Get latest service agreement: No minted assets on blockchain: ${blockchain}`, + ); + return; + } + throw error; + } + + const latestDbTokenId = + (await this.repositoryModuleManager.getLatestServiceAgreementTokenId(blockchain)) ?? 0; + + this.logger.debug( + `Get latest service agreement: Latest token id on chain: ${latestBlockchainTokenId}, latest token id in database: ${latestDbTokenId} on blockchain: ${blockchain}`, + ); + + const missingTokenIds = Array.from( + { length: latestBlockchainTokenId - latestDbTokenId }, + (_, index) => latestDbTokenId + index + 1, + ); + this.logger.debug( + `Get latest service agreement: Found ${missingTokenIds.length} on blockchain: ${blockchain}`, + ); + let batchNumber = 0; + while (batchNumber * BATCH_SIZE < missingTokenIds.length) { + const promises = []; + for ( + let i = batchNumber * BATCH_SIZE; + i < missingTokenIds.length && i < (batchNumber + 1) * BATCH_SIZE; + i += 1 + ) { + const tokenIdToBeFetched = missingTokenIds[i]; + promises.push( + this.getAgreementDataForToken(tokenIdToBeFetched, blockchain, contract), + ); + } + + // eslint-disable-next-line no-await-in-loop + const missingAgreements = await Promise.all(promises); + + // eslint-disable-next-line no-await-in-loop + await this.repositoryModuleManager.bulkCreateServiceAgreementRecords( + missingAgreements.filter((agreement) => agreement != null), + ); + batchNumber += 1; + } + if (missingTokenIds.length !== 0) { + this.logger.debug( + `Get latest service agreement: Successfully fetched ${missingTokenIds.length} on blockchain: ${blockchain}`, + ); + } + } + + async getAgreementDataForToken( + tokenId, + blockchain, + contract, + hashFunctionId = CONTENT_ASSET_HASH_FUNCTION_ID, + ) { + this.logger.debug( + `Get latest service agreement: Getting agreement data for token id: ${tokenId} on blockchain: ${blockchain}`, + ); + const assertionIds = await this.blockchainModuleManager.getAssertionIds( + blockchain, + contract, + tokenId, + ); + const keyword = await this.ualService.calculateLocationKeyword( + blockchain, + contract, + tokenId, + assertionIds[0], + ); + const agreementId = await this.serviceAgreementService.generateId( + blockchain, + contract, + tokenId, + keyword, + hashFunctionId, + ); + const agreementData = await this.blockchainModuleManager.getAgreementData( + blockchain, + agreementId, + ); + + if (!agreementData) { + this.logger.warn( + `Unable to fetch agreement data while processing asset created event for agreement id: ${agreementId}, blockchain id: ${blockchain}`, + ); + } + + const latestStateIndex = assertionIds.length - 1; + + return { + blockchainId: blockchain, + assetStorageContractAddress: contract, + tokenId, + agreementId, + startTime: agreementData.startTime, + epochsNumber: agreementData.epochsNumber, + epochLength: agreementData.epochLength, + scoreFunctionId: agreementData.scoreFunctionId, + stateIndex: latestStateIndex, + assertionId: assertionIds[latestStateIndex], + hashFunctionId, + keyword, + proofWindowOffsetPerc: agreementData.proofWindowOffsetPerc, + dataSource: SERVICE_AGREEMENT_SOURCES.NODE, + }; + } + + /** + * Recover system from failure + * @param error + */ + async recover() { + return Command.repeat(); + } + + /** + * Builds default command + * @param map + * @returns {{add, data: *, delay: *, deadline: *}} + */ + default(map) { + const command = { + name: 'blockchainGetLatestServiceAgreement', + data: {}, + period: GET_LATEST_SERVICE_AGREEMENT_FREQUENCY_MILLS, + transactional: false, + }; + Object.assign(command, map); + return command; + } +} + +export default BlockchainGetLatestServiceAgreement; diff --git a/src/commands/common/get-latest-service-agreement/get-latest-service-agreement.js b/src/commands/common/get-latest-service-agreement/get-latest-service-agreement.js new file mode 100644 index 000000000..c95153463 --- /dev/null +++ b/src/commands/common/get-latest-service-agreement/get-latest-service-agreement.js @@ -0,0 +1,71 @@ +import Command from '../../command.js'; +import { GET_LATEST_SERVICE_AGREEMENT_FREQUENCY_MILLS } from '../../../constants/constants.js'; + +class GetLatestServiceAgreement extends Command { + constructor(ctx) { + super(ctx); + this.commandExecutor = ctx.commandExecutor; + this.shardingTableService = ctx.shardingTableService; + this.repositoryModuleManager = ctx.repositoryModuleManager; + this.blockchainModuleManager = ctx.blockchainModuleManager; + } + + /** + * Executes command and produces one or more events + * @param command + */ + async execute() { + const operationId = this.operationIdService.generateId(); + + this.logger.info( + `Get latest service agreement: Starting get latest service agreement command for operation id: ${operationId}`, + ); + + await this.commandExecutor.delete('blockchainGetLatestServiceAgreement'); + + await Promise.all( + this.blockchainModuleManager.getImplementationNames().map(async (blockchain) => { + const commandData = { + blockchain, + operationId, + }; + + return this.commandExecutor.add({ + name: 'blockchainGetLatestServiceAgreement', + data: commandData, + period: GET_LATEST_SERVICE_AGREEMENT_FREQUENCY_MILLS, + }); + }), + ); + + return Command.empty(); + } + + /** + * Recover system from failure + * @param command + * @param error + */ + async recover(command) { + this.logger.warn(`Failed to execute ${command.name}. Error: ${command.message}`); + + return Command.repeat(); + } + + /** + * Builds default command + * @param map + * @returns {{add, data: *, delay: *, deadline: *}} + */ + default(map) { + const command = { + name: 'getLatestServiceAgreement', + data: {}, + transactional: false, + }; + Object.assign(command, map); + return command; + } +} + +export default GetLatestServiceAgreement; diff --git a/src/commands/protocols/common/epoch-check/blockchain-epoch-check-command.js b/src/commands/protocols/common/epoch-check/blockchain-epoch-check-command.js index 8cc0e6a5e..baea3a9ec 100644 --- a/src/commands/protocols/common/epoch-check/blockchain-epoch-check-command.js +++ b/src/commands/protocols/common/epoch-check/blockchain-epoch-check-command.js @@ -493,12 +493,9 @@ class BlockchainEpochCheckCommand extends Command { /** * Recover system from failure - * @param command * @param error */ - async recover(command) { - this.logger.warn(`Failed to execute ${command.name}. Error: ${command.message}`); - + async recover() { return Command.repeat(); } diff --git a/src/constants/constants.js b/src/constants/constants.js index 46d710dbd..66f7ad18a 100644 --- a/src/constants/constants.js +++ b/src/constants/constants.js @@ -160,6 +160,8 @@ export const REMOVE_SESSION_COMMAND_DELAY = 2 * 60 * 1000; export const OPERATION_IDS_COMMAND_CLEANUP_TIME_MILLS = 24 * 60 * 60 * 1000; +export const GET_LATEST_SERVICE_AGREEMENT_FREQUENCY_MILLS = 30 * 1000; + export const DIAL_PEERS_COMMAND_FREQUENCY_MILLS = 30 * 1000; export const DIAL_PEERS_CONCURRENCY = 10; @@ -174,6 +176,7 @@ export const PERMANENT_COMMANDS = [ 'commandsCleanerCommand', 'dialPeersCommand', 'epochCheckCommand', + 'getLatestServiceAgreement', 'blockchainEventCleanerCommand', 'getCleanerCommand', 'getResponseCleanerCommand', @@ -464,6 +467,7 @@ export const EXPECTED_TRANSACTION_ERRORS = { PROOF_WINDOW_CLOSED: 'ProofWindowClosed', NODE_NOT_AWARDED: 'NodeNotAwarded', WRONG_MERKLE_PROOF: 'WrongMerkleProof', + NO_MINTED_ASSETS: 'NoMintedAssets', }; /** @@ -661,24 +665,14 @@ export const CONTRACT_EVENTS = { SHARDING_TABLE: ['NodeAdded', 'NodeRemoved'], STAKING: ['StakeIncreased', 'StakeWithdrawalStarted'], PROFILE: ['AskUpdated'], - CONTENT_ASSET: ['AssetMinted'], COMMIT_MANAGER_V1: ['StateFinalized'], - SERVICE_AGREEMENT_V1: [ - 'ServiceAgreementV1Created', - 'ServiceAgreementV1Extended', - 'ServiceAgreementV1Terminated', - ], + SERVICE_AGREEMENT_V1: ['ServiceAgreementV1Extended', 'ServiceAgreementV1Terminated'], PARAMETERS_STORAGE: ['ParameterChanged'], LOG2PLDSF: ['ParameterChanged'], LINEAR_SUM: ['ParameterChanged'], }; -export const GROUPED_CONTRACT_EVENTS = { - AssetCreatedGroup: { - events: ['AssetMinted', 'ServiceAgreementV1Created'], - groupingKey: 'tokenId', - }, -}; +export const GROUPED_CONTRACT_EVENTS = {}; export const CONTRACT_EVENT_TO_GROUP_MAPPING = (() => { const mapping = {}; diff --git a/src/controllers/http-api/v0/bid-suggestion-http-api-controller-v0.js b/src/controllers/http-api/v0/bid-suggestion-http-api-controller-v0.js index e549db4bc..6e23e9877 100644 --- a/src/controllers/http-api/v0/bid-suggestion-http-api-controller-v0.js +++ b/src/controllers/http-api/v0/bid-suggestion-http-api-controller-v0.js @@ -53,13 +53,8 @@ class BidSuggestionController extends BaseController { } = req.query; let { proximityScoreFunctionsPairId, bidSuggestionRange } = req.query; try { - // TODO: ADD-DOCS - if (!proximityScoreFunctionsPairId) { - if (blockchain.startsWith('otp') || blockchain.startsWith('hardhat1')) - proximityScoreFunctionsPairId = 1; - else if (blockchain.startsWith('gnosis') || blockchain.startsWith('hardhat2')) - proximityScoreFunctionsPairId = 2; - } + proximityScoreFunctionsPairId = proximityScoreFunctionsPairId ?? 2; + if (!bidSuggestionRange) { bidSuggestionRange = LOW_BID_SUGGESTION; } diff --git a/src/modules/blockchain/implementation/hardhat/hardhat-service.js b/src/modules/blockchain/implementation/hardhat/hardhat-service.js index 315e52644..5a89c03a2 100644 --- a/src/modules/blockchain/implementation/hardhat/hardhat-service.js +++ b/src/modules/blockchain/implementation/hardhat/hardhat-service.js @@ -26,9 +26,6 @@ class HardhatService extends Web3Service { } async getAgreementScoreFunctionId() { - if (this.getBlockchainId() === 'hardhat1:31337') { - return 1; - } return 2; } } diff --git a/src/modules/blockchain/implementation/ot-parachain/ot-parachain-service.js b/src/modules/blockchain/implementation/ot-parachain/ot-parachain-service.js index 37ba3fa3c..7ba9c69b4 100644 --- a/src/modules/blockchain/implementation/ot-parachain/ot-parachain-service.js +++ b/src/modules/blockchain/implementation/ot-parachain/ot-parachain-service.js @@ -201,10 +201,6 @@ class OtParachainService extends Web3Service { }); return wallets; } - - async getAgreementScoreFunctionId() { - return 1; - } } export default OtParachainService; diff --git a/src/modules/repository/implementation/sequelize/repositories/service-agreement-repository.js b/src/modules/repository/implementation/sequelize/repositories/service-agreement-repository.js index 57e8ef910..562f9d061 100644 --- a/src/modules/repository/implementation/sequelize/repositories/service-agreement-repository.js +++ b/src/modules/repository/implementation/sequelize/repositories/service-agreement-repository.js @@ -176,7 +176,10 @@ class ServiceAgreementRepository { [Sequelize.Op.gt]: Sequelize.literal(currentEpoch), }, }, - order: [[Sequelize.col('timeLeftInSubmitCommitWindow'), 'ASC']], + order: [ + ['scoreFunctionId', 'DESC'], + [Sequelize.col('timeLeftInSubmitCommitWindow'), 'ASC'], + ], limit: 100, raw: true, }); @@ -232,7 +235,10 @@ class ServiceAgreementRepository { [Sequelize.Op.gt]: Sequelize.literal(currentEpoch), }, }, - order: [[Sequelize.col('timeLeftInSubmitProofWindow'), 'ASC']], + order: [ + ['scoreFunctionId', 'DESC'], + [Sequelize.col('timeLeftInSubmitProofWindow'), 'ASC'], + ], limit: 100, raw: true, }); @@ -262,6 +268,14 @@ class ServiceAgreementRepository { order: [['token_id', 'asc']], }); } + + async getLatestServiceAgreementTokenId(blockchainId) { + return this.model.max('tokenId', { + where: { + blockchainId, + }, + }); + } } export default ServiceAgreementRepository; diff --git a/src/modules/repository/repository-module-manager.js b/src/modules/repository/repository-module-manager.js index 7e1498e83..e39a44b81 100644 --- a/src/modules/repository/repository-module-manager.js +++ b/src/modules/repository/repository-module-manager.js @@ -449,6 +449,12 @@ class RepositoryModuleManager extends BaseModuleManager { blockchainId, ); } + + async getLatestServiceAgreementTokenId(blockchainId) { + return this.getRepository('service_agreement').getLatestServiceAgreementTokenId( + blockchainId, + ); + } } export default RepositoryModuleManager; diff --git a/src/service/blockchain-event-listener-service.js b/src/service/blockchain-event-listener-service.js index 005d209ce..62c230bd5 100644 --- a/src/service/blockchain-event-listener-service.js +++ b/src/service/blockchain-event-listener-service.js @@ -11,7 +11,6 @@ import { DELAY_BETWEEN_FAILED_FETCH_EVENTS_MILLIS, CONTRACT_EVENT_TO_GROUP_MAPPING, GROUPED_CONTRACT_EVENTS, - SERVICE_AGREEMENT_SOURCES, } from '../constants/constants.js'; const fetchEventsFailedCount = {}; @@ -73,12 +72,6 @@ class BlockchainEventListenerService { } const syncContractEventsPromises = [ - this.getContractEvents( - blockchainId, - CONTRACTS.CONTENT_ASSET, - currentBlock, - CONTRACT_EVENTS.CONTENT_ASSET, - ), this.getContractEvents( blockchainId, CONTRACTS.SHARDING_TABLE_CONTRACT, @@ -520,86 +513,6 @@ class BlockchainEventListenerService { ); } - async handleAssetCreatedGroupEvents(blockGroupEvents) { - await Promise.all( - blockGroupEvents.map(async (eventsGroup) => { - // Parse and combine Arguments of both AssetMinted and ServiceAgreementCreated Events - const combinedData = eventsGroup.reduce((accumulator, event) => { - try { - const eventData = JSON.parse(event.data); - return { - ...accumulator, - ...eventData, - blockchainId: event.blockchainId, - }; - } catch (error) { - this.logger.error(`Error parsing event data: ${error}`); - return accumulator; - } - }, {}); - - const { - blockchainId, - assetContract: contract, - tokenId, - keyword, - hashFunctionId, - state: assertionId, - startTime, - epochsNumber, - epochLength, - // TODO: Uncomment when these arguments are added to the ServiceAgreementV1Created event - // scoreFunctionId, - // proofWindowOffsetPerc, - } = combinedData; - - const agreementId = this.serviceAgreementService.generateId( - blockchainId, - contract, - tokenId, - keyword, - hashFunctionId, - ); - - const agreementRecord = - await this.repositoryModuleManager.getServiceAgreementRecord(agreementId); - if (agreementRecord) { - this.logger.trace( - `Skipping processing of asset created event, agreement data present in database for agreement id: ${agreementId} on blockchain ${blockchainId}`, - ); - } else { - const agreementData = await this.blockchainModuleManager.getAgreementData( - blockchainId, - agreementId, - ); - - if (!agreementData) { - this.logger.warn( - `Unable to fetch agreement data while processing asset created event for agreement id: ${agreementId}, blockchain id: ${blockchainId}`, - ); - } - - await this.repositoryModuleManager.updateServiceAgreementRecord( - blockchainId, - contract, - tokenId, - agreementId, - startTime, - epochsNumber, - epochLength, - agreementData?.scoreFunctionId ?? 0, - agreementData?.proofWindowOffsetPerc ?? 0, - hashFunctionId, - keyword, - assertionId, - 0, - SERVICE_AGREEMENT_SOURCES.EVENT, - ); - } - }), - ); - } - async handleServiceAgreementV1ExtendedEvents(blockEvents) { await Promise.all( blockEvents.map(async (event) => { diff --git a/tools/local-network-setup/setup-macos-environment.sh b/tools/local-network-setup/setup-macos-environment.sh index 77ddff452..b7f0dba41 100755 --- a/tools/local-network-setup/setup-macos-environment.sh +++ b/tools/local-network-setup/setup-macos-environment.sh @@ -51,7 +51,7 @@ then osascript -e "tell app \"Terminal\" do script \"cd $pathToOtNode - node tools/local-network-setup/run-local-blockchain.js 8545 :v1\" + node tools/local-network-setup/run-local-blockchain.js 8545 :v2\" end tell" echo Waiting for hardhat to start and contracts deployment From ba2a136045c01464f4416aa51b31cd72d9988684 Mon Sep 17 00:00:00 2001 From: djordjekovac Date: Fri, 12 Apr 2024 13:23:53 +0200 Subject: [PATCH 02/13] Optimized function for fetching latest service agreements. (#3125) * Optimized function for fetching latest service agreements. * Removed old functino for creating array --- ...blockchain-get-latest-service-agreement.js | 53 +++++++++---------- 1 file changed, 25 insertions(+), 28 deletions(-) diff --git a/src/commands/common/get-latest-service-agreement/blockchain-get-latest-service-agreement.js b/src/commands/common/get-latest-service-agreement/blockchain-get-latest-service-agreement.js index a2085b706..80ec73e83 100644 --- a/src/commands/common/get-latest-service-agreement/blockchain-get-latest-service-agreement.js +++ b/src/commands/common/get-latest-service-agreement/blockchain-get-latest-service-agreement.js @@ -63,39 +63,36 @@ class BlockchainGetLatestServiceAgreement extends Command { `Get latest service agreement: Latest token id on chain: ${latestBlockchainTokenId}, latest token id in database: ${latestDbTokenId} on blockchain: ${blockchain}`, ); - const missingTokenIds = Array.from( - { length: latestBlockchainTokenId - latestDbTokenId }, - (_, index) => latestDbTokenId + index + 1, - ); - this.logger.debug( - `Get latest service agreement: Found ${missingTokenIds.length} on blockchain: ${blockchain}`, - ); - let batchNumber = 0; - while (batchNumber * BATCH_SIZE < missingTokenIds.length) { - const promises = []; - for ( - let i = batchNumber * BATCH_SIZE; - i < missingTokenIds.length && i < (batchNumber + 1) * BATCH_SIZE; - i += 1 + let tokenIdDifference = latestBlockchainTokenId - latestDbTokenId; + let getAgreementDataPromise = []; + for ( + let tokenIdToBeFetched = latestDbTokenId + 1; + tokenIdToBeFetched <= latestBlockchainTokenId; + tokenIdToBeFetched += 1 + ) { + getAgreementDataPromise.push( + this.getAgreementDataForToken(tokenIdToBeFetched, blockchain, contract), + ); + if ( + getAgreementDataPromise.length === tokenIdDifference || + getAgreementDataPromise.length === BATCH_SIZE ) { - const tokenIdToBeFetched = missingTokenIds[i]; - promises.push( - this.getAgreementDataForToken(tokenIdToBeFetched, blockchain, contract), + // eslint-disable-next-line no-await-in-loop + const missingAgreements = await Promise.all(getAgreementDataPromise); + + // eslint-disable-next-line no-await-in-loop + await this.repositoryModuleManager.bulkCreateServiceAgreementRecords( + missingAgreements.filter((agreement) => agreement != null), ); + getAgreementDataPromise = []; + tokenIdDifference -= BATCH_SIZE; } - - // eslint-disable-next-line no-await-in-loop - const missingAgreements = await Promise.all(promises); - - // eslint-disable-next-line no-await-in-loop - await this.repositoryModuleManager.bulkCreateServiceAgreementRecords( - missingAgreements.filter((agreement) => agreement != null), - ); - batchNumber += 1; } - if (missingTokenIds.length !== 0) { + if (latestBlockchainTokenId - latestDbTokenId !== 0) { this.logger.debug( - `Get latest service agreement: Successfully fetched ${missingTokenIds.length} on blockchain: ${blockchain}`, + `Get latest service agreement: Successfully fetched ${ + latestBlockchainTokenId - latestDbTokenId + } on blockchain: ${blockchain}`, ); } } From be561960d1e76574860bcd230dbcbf7cacf23ef7 Mon Sep 17 00:00:00 2001 From: djordjekovac Date: Fri, 12 Apr 2024 13:28:35 +0200 Subject: [PATCH 03/13] Version bumped (#3127) --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 673c2e145..3a18c9d49 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "origintrail_node", - "version": "6.2.4", + "version": "6.3.0", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "origintrail_node", - "version": "6.2.4", + "version": "6.3.0", "license": "ISC", "dependencies": { "@comunica/query-sparql": "^2.4.3", diff --git a/package.json b/package.json index 1eae3e6d3..57fad629e 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "origintrail_node", - "version": "6.2.4", + "version": "6.3.0", "description": "OTNode V6", "main": "index.js", "type": "module", From 7e7d95e7cd2abbc623923f48572af11c9e40e309 Mon Sep 17 00:00:00 2001 From: Mihajlo Pavlovic Date: Wed, 17 Apr 2024 14:14:06 +0200 Subject: [PATCH 04/13] Set proximityScoreFunctionsPairId in bid suggestion schema to accept only 2 (#3129) * Set proximityScoreFunctionsPairId in bid suggestion schema to accept only 2 * Change how proximityScoreFunctionsPairId value is limited --- .../http-api/v0/request-schema/bid-suggestion-schema-v0.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/controllers/http-api/v0/request-schema/bid-suggestion-schema-v0.js b/src/controllers/http-api/v0/request-schema/bid-suggestion-schema-v0.js index 80e5baaa4..f6c72aea0 100644 --- a/src/controllers/http-api/v0/request-schema/bid-suggestion-schema-v0.js +++ b/src/controllers/http-api/v0/request-schema/bid-suggestion-schema-v0.js @@ -39,7 +39,7 @@ export default (argumentsObject) => ({ }, proximityScoreFunctionsPairId: { type: 'number', - minimum: 1, + minimum: 2, maximum: 2, }, bidSuggestionRange: { From 64e887f1533ee7c1bf2019900ad0fc527dda8c5c Mon Sep 17 00:00:00 2001 From: Djordje Kovacevic Date: Wed, 17 Apr 2024 16:06:39 +0200 Subject: [PATCH 05/13] Always using new scoring function when calling bid suggestion --- .../http-api/v0/bid-suggestion-http-api-controller-v0.js | 4 ++-- .../http-api/v0/request-schema/bid-suggestion-schema-v0.js | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/controllers/http-api/v0/bid-suggestion-http-api-controller-v0.js b/src/controllers/http-api/v0/bid-suggestion-http-api-controller-v0.js index 6e23e9877..93fe75c81 100644 --- a/src/controllers/http-api/v0/bid-suggestion-http-api-controller-v0.js +++ b/src/controllers/http-api/v0/bid-suggestion-http-api-controller-v0.js @@ -51,9 +51,9 @@ class BidSuggestionController extends BaseController { firstAssertionId, hashFunctionId, } = req.query; - let { proximityScoreFunctionsPairId, bidSuggestionRange } = req.query; + let { bidSuggestionRange } = req.query; try { - proximityScoreFunctionsPairId = proximityScoreFunctionsPairId ?? 2; + const proximityScoreFunctionsPairId = 2; if (!bidSuggestionRange) { bidSuggestionRange = LOW_BID_SUGGESTION; diff --git a/src/controllers/http-api/v0/request-schema/bid-suggestion-schema-v0.js b/src/controllers/http-api/v0/request-schema/bid-suggestion-schema-v0.js index f6c72aea0..80e5baaa4 100644 --- a/src/controllers/http-api/v0/request-schema/bid-suggestion-schema-v0.js +++ b/src/controllers/http-api/v0/request-schema/bid-suggestion-schema-v0.js @@ -39,7 +39,7 @@ export default (argumentsObject) => ({ }, proximityScoreFunctionsPairId: { type: 'number', - minimum: 2, + minimum: 1, maximum: 2, }, bidSuggestionRange: { From 4b4b2bf435c287e440d234c3d7f4a3b816fb311d Mon Sep 17 00:00:00 2001 From: Djordje Kovacevic Date: Fri, 19 Apr 2024 11:39:10 +0200 Subject: [PATCH 06/13] Added retry logic for agreement ids --- package-lock.json | 4 +- package.json | 2 +- ...blockchain-get-latest-service-agreement.js | 130 +++++++++++------- 3 files changed, 80 insertions(+), 56 deletions(-) diff --git a/package-lock.json b/package-lock.json index 3a18c9d49..1690bab2e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "origintrail_node", - "version": "6.3.0", + "version": "6.3.0+hotfix.1", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "origintrail_node", - "version": "6.3.0", + "version": "6.3.0+hotfix.1", "license": "ISC", "dependencies": { "@comunica/query-sparql": "^2.4.3", diff --git a/package.json b/package.json index 57fad629e..829f67112 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "origintrail_node", - "version": "6.3.0", + "version": "6.3.0+hotfix.1", "description": "OTNode V6", "main": "index.js", "type": "module", diff --git a/src/commands/common/get-latest-service-agreement/blockchain-get-latest-service-agreement.js b/src/commands/common/get-latest-service-agreement/blockchain-get-latest-service-agreement.js index 80ec73e83..a2619ad46 100644 --- a/src/commands/common/get-latest-service-agreement/blockchain-get-latest-service-agreement.js +++ b/src/commands/common/get-latest-service-agreement/blockchain-get-latest-service-agreement.js @@ -1,3 +1,5 @@ +/* eslint-disable no-await-in-loop */ +import { setTimeout as sleep } from 'timers/promises'; import Command from '../../command.js'; import { CONTENT_ASSET_HASH_FUNCTION_ID, @@ -7,6 +9,8 @@ import { } from '../../../constants/constants.js'; const BATCH_SIZE = 50; +const MAX_RETRY_COUNT = 5; +const RETRY_DELAY_IN_SECONDS = 2; class BlockchainGetLatestServiceAgreement extends Command { constructor(ctx) { @@ -42,9 +46,8 @@ class BlockchainGetLatestServiceAgreement extends Command { ); let latestBlockchainTokenId; try { - latestBlockchainTokenId = await this.blockchainModuleManager.getLatestTokenId( - blockchain, - contract, + latestBlockchainTokenId = Number( + await this.blockchainModuleManager.getLatestTokenId(blockchain, contract), ); } catch (error) { if (error.message.includes(EXPECTED_TRANSACTION_ERRORS.NO_MINTED_ASSETS)) { @@ -53,7 +56,10 @@ class BlockchainGetLatestServiceAgreement extends Command { ); return; } - throw error; + this.logger.error( + `Unable to process agreement data for asset contract ${contract}. Error: ${error}`, + ); + return; } const latestDbTokenId = @@ -77,10 +83,8 @@ class BlockchainGetLatestServiceAgreement extends Command { getAgreementDataPromise.length === tokenIdDifference || getAgreementDataPromise.length === BATCH_SIZE ) { - // eslint-disable-next-line no-await-in-loop const missingAgreements = await Promise.all(getAgreementDataPromise); - // eslint-disable-next-line no-await-in-loop await this.repositoryModuleManager.bulkCreateServiceAgreementRecords( missingAgreements.filter((agreement) => agreement != null), ); @@ -103,56 +107,76 @@ class BlockchainGetLatestServiceAgreement extends Command { contract, hashFunctionId = CONTENT_ASSET_HASH_FUNCTION_ID, ) { - this.logger.debug( - `Get latest service agreement: Getting agreement data for token id: ${tokenId} on blockchain: ${blockchain}`, - ); - const assertionIds = await this.blockchainModuleManager.getAssertionIds( - blockchain, - contract, - tokenId, - ); - const keyword = await this.ualService.calculateLocationKeyword( - blockchain, - contract, - tokenId, - assertionIds[0], - ); - const agreementId = await this.serviceAgreementService.generateId( - blockchain, - contract, - tokenId, - keyword, - hashFunctionId, - ); - const agreementData = await this.blockchainModuleManager.getAgreementData( - blockchain, - agreementId, - ); + try { + this.logger.debug( + `Get latest service agreement: Getting agreement data for token id: ${tokenId} on blockchain: ${blockchain}`, + ); + let assertionIds = []; + let retryCount = 0; + + while (assertionIds.length === 0) { + if (retryCount === MAX_RETRY_COUNT) { + throw Error( + `Get latest service agreement: Unable to get assertion ids for token id: ${tokenId} on blockchain: ${blockchain}`, + ); + } + this.logger.debug( + `Get latest service agreement: getting assertion ids retry ${retryCount} for token id: ${tokenId} on blockchain: ${blockchain}`, + ); + assertionIds = await this.blockchainModuleManager.getAssertionIds( + blockchain, + contract, + tokenId, + ); + retryCount += 1; + await sleep(RETRY_DELAY_IN_SECONDS * 1000); + } - if (!agreementData) { - this.logger.warn( - `Unable to fetch agreement data while processing asset created event for agreement id: ${agreementId}, blockchain id: ${blockchain}`, + const keyword = await this.ualService.calculateLocationKeyword( + blockchain, + contract, + tokenId, + assertionIds[0], + ); + const agreementId = await this.serviceAgreementService.generateId( + blockchain, + contract, + tokenId, + keyword, + hashFunctionId, + ); + const agreementData = await this.blockchainModuleManager.getAgreementData( + blockchain, + agreementId, ); - } - const latestStateIndex = assertionIds.length - 1; - - return { - blockchainId: blockchain, - assetStorageContractAddress: contract, - tokenId, - agreementId, - startTime: agreementData.startTime, - epochsNumber: agreementData.epochsNumber, - epochLength: agreementData.epochLength, - scoreFunctionId: agreementData.scoreFunctionId, - stateIndex: latestStateIndex, - assertionId: assertionIds[latestStateIndex], - hashFunctionId, - keyword, - proofWindowOffsetPerc: agreementData.proofWindowOffsetPerc, - dataSource: SERVICE_AGREEMENT_SOURCES.NODE, - }; + if (!agreementData) { + throw Error( + `Get latest service agreement: Unable to fetch agreement data while processing asset created event for agreement id: ${agreementId}, blockchain id: ${blockchain}`, + ); + } + + const latestStateIndex = assertionIds.length - 1; + + return { + blockchainId: blockchain, + assetStorageContractAddress: contract, + tokenId, + agreementId, + startTime: agreementData.startTime, + epochsNumber: agreementData.epochsNumber, + epochLength: agreementData.epochLength, + scoreFunctionId: agreementData.scoreFunctionId, + stateIndex: latestStateIndex, + assertionId: assertionIds[latestStateIndex], + hashFunctionId, + keyword, + proofWindowOffsetPerc: agreementData.proofWindowOffsetPerc, + dataSource: SERVICE_AGREEMENT_SOURCES.NODE, + }; + } catch (error) { + this.logger.error(error.message); + } } /** From dfb9c5f00b2d6ece9e00bf6fb5743b6f81249ae3 Mon Sep 17 00:00:00 2001 From: Djordje Kovacevic Date: Fri, 19 Apr 2024 12:35:27 +0200 Subject: [PATCH 07/13] PR comments fixed --- ...blockchain-get-latest-service-agreement.js | 29 ++++++++++++------- src/constants/constants.js | 4 +++ 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/src/commands/common/get-latest-service-agreement/blockchain-get-latest-service-agreement.js b/src/commands/common/get-latest-service-agreement/blockchain-get-latest-service-agreement.js index a2619ad46..3ee827b17 100644 --- a/src/commands/common/get-latest-service-agreement/blockchain-get-latest-service-agreement.js +++ b/src/commands/common/get-latest-service-agreement/blockchain-get-latest-service-agreement.js @@ -4,14 +4,14 @@ import Command from '../../command.js'; import { CONTENT_ASSET_HASH_FUNCTION_ID, EXPECTED_TRANSACTION_ERRORS, + GET_ASSERTION_IDS_MAX_RETRY_COUNT, + GET_ASSERTION_IDS_RETRY_DELAY_IN_SECONDS, + GET_LATEST_SERVICE_AGREEMENT_BATCH_SIZE, + GET_LATEST_SERVICE_AGREEMENT_EXCLUDE_LATEST_TOKEN_ID, GET_LATEST_SERVICE_AGREEMENT_FREQUENCY_MILLS, SERVICE_AGREEMENT_SOURCES, } from '../../../constants/constants.js'; -const BATCH_SIZE = 50; -const MAX_RETRY_COUNT = 5; -const RETRY_DELAY_IN_SECONDS = 2; - class BlockchainGetLatestServiceAgreement extends Command { constructor(ctx) { super(ctx); @@ -46,9 +46,9 @@ class BlockchainGetLatestServiceAgreement extends Command { ); let latestBlockchainTokenId; try { - latestBlockchainTokenId = Number( - await this.blockchainModuleManager.getLatestTokenId(blockchain, contract), - ); + latestBlockchainTokenId = + Number(await this.blockchainModuleManager.getLatestTokenId(blockchain, contract)) - + GET_LATEST_SERVICE_AGREEMENT_EXCLUDE_LATEST_TOKEN_ID; } catch (error) { if (error.message.includes(EXPECTED_TRANSACTION_ERRORS.NO_MINTED_ASSETS)) { this.logger.info( @@ -65,6 +65,13 @@ class BlockchainGetLatestServiceAgreement extends Command { const latestDbTokenId = (await this.repositoryModuleManager.getLatestServiceAgreementTokenId(blockchain)) ?? 0; + if (latestBlockchainTokenId < latestDbTokenId) { + this.logger.debug( + `Get latest service agreement: No new agreements found on blockchain: ${blockchain}.`, + ); + return; + } + this.logger.debug( `Get latest service agreement: Latest token id on chain: ${latestBlockchainTokenId}, latest token id in database: ${latestDbTokenId} on blockchain: ${blockchain}`, ); @@ -81,7 +88,7 @@ class BlockchainGetLatestServiceAgreement extends Command { ); if ( getAgreementDataPromise.length === tokenIdDifference || - getAgreementDataPromise.length === BATCH_SIZE + getAgreementDataPromise.length === GET_LATEST_SERVICE_AGREEMENT_BATCH_SIZE ) { const missingAgreements = await Promise.all(getAgreementDataPromise); @@ -89,7 +96,7 @@ class BlockchainGetLatestServiceAgreement extends Command { missingAgreements.filter((agreement) => agreement != null), ); getAgreementDataPromise = []; - tokenIdDifference -= BATCH_SIZE; + tokenIdDifference -= GET_LATEST_SERVICE_AGREEMENT_BATCH_SIZE; } } if (latestBlockchainTokenId - latestDbTokenId !== 0) { @@ -115,7 +122,7 @@ class BlockchainGetLatestServiceAgreement extends Command { let retryCount = 0; while (assertionIds.length === 0) { - if (retryCount === MAX_RETRY_COUNT) { + if (retryCount === GET_ASSERTION_IDS_MAX_RETRY_COUNT) { throw Error( `Get latest service agreement: Unable to get assertion ids for token id: ${tokenId} on blockchain: ${blockchain}`, ); @@ -129,7 +136,7 @@ class BlockchainGetLatestServiceAgreement extends Command { tokenId, ); retryCount += 1; - await sleep(RETRY_DELAY_IN_SECONDS * 1000); + await sleep(GET_ASSERTION_IDS_RETRY_DELAY_IN_SECONDS * 1000); } const keyword = await this.ualService.calculateLocationKeyword( diff --git a/src/constants/constants.js b/src/constants/constants.js index 66f7ad18a..65c409ff7 100644 --- a/src/constants/constants.js +++ b/src/constants/constants.js @@ -548,6 +548,10 @@ export const ARCHIVE_UPDATE_RESPONSES_FOLDER = 'update_responses'; */ export const COMMAND_QUEUE_PARALLELISM = 100; +export const GET_LATEST_SERVICE_AGREEMENT_BATCH_SIZE = 50; +export const GET_ASSERTION_IDS_MAX_RETRY_COUNT = 5; +export const GET_ASSERTION_IDS_RETRY_DELAY_IN_SECONDS = 2; +export const GET_LATEST_SERVICE_AGREEMENT_EXCLUDE_LATEST_TOKEN_ID = 10; /** * @constant {object} HTTP_API_ROUTES - * HTTP API Routes with parameters From b6e91a97cd3bc7abfedef78f08a77cd974ce2490 Mon Sep 17 00:00:00 2001 From: djordjekovac Date: Fri, 19 Apr 2024 13:55:27 +0200 Subject: [PATCH 08/13] Devnet 4.3.0 hotfix 1 (#3136) * Added retry logic for agreement ids * PR comments fixed --- package-lock.json | 4 +- package.json | 2 +- ...blockchain-get-latest-service-agreement.js | 147 +++++++++++------- src/constants/constants.js | 4 + 4 files changed, 96 insertions(+), 61 deletions(-) diff --git a/package-lock.json b/package-lock.json index 3a18c9d49..1690bab2e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "origintrail_node", - "version": "6.3.0", + "version": "6.3.0+hotfix.1", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "origintrail_node", - "version": "6.3.0", + "version": "6.3.0+hotfix.1", "license": "ISC", "dependencies": { "@comunica/query-sparql": "^2.4.3", diff --git a/package.json b/package.json index 57fad629e..829f67112 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "origintrail_node", - "version": "6.3.0", + "version": "6.3.0+hotfix.1", "description": "OTNode V6", "main": "index.js", "type": "module", diff --git a/src/commands/common/get-latest-service-agreement/blockchain-get-latest-service-agreement.js b/src/commands/common/get-latest-service-agreement/blockchain-get-latest-service-agreement.js index 80ec73e83..3ee827b17 100644 --- a/src/commands/common/get-latest-service-agreement/blockchain-get-latest-service-agreement.js +++ b/src/commands/common/get-latest-service-agreement/blockchain-get-latest-service-agreement.js @@ -1,13 +1,17 @@ +/* eslint-disable no-await-in-loop */ +import { setTimeout as sleep } from 'timers/promises'; import Command from '../../command.js'; import { CONTENT_ASSET_HASH_FUNCTION_ID, EXPECTED_TRANSACTION_ERRORS, + GET_ASSERTION_IDS_MAX_RETRY_COUNT, + GET_ASSERTION_IDS_RETRY_DELAY_IN_SECONDS, + GET_LATEST_SERVICE_AGREEMENT_BATCH_SIZE, + GET_LATEST_SERVICE_AGREEMENT_EXCLUDE_LATEST_TOKEN_ID, GET_LATEST_SERVICE_AGREEMENT_FREQUENCY_MILLS, SERVICE_AGREEMENT_SOURCES, } from '../../../constants/constants.js'; -const BATCH_SIZE = 50; - class BlockchainGetLatestServiceAgreement extends Command { constructor(ctx) { super(ctx); @@ -42,10 +46,9 @@ class BlockchainGetLatestServiceAgreement extends Command { ); let latestBlockchainTokenId; try { - latestBlockchainTokenId = await this.blockchainModuleManager.getLatestTokenId( - blockchain, - contract, - ); + latestBlockchainTokenId = + Number(await this.blockchainModuleManager.getLatestTokenId(blockchain, contract)) - + GET_LATEST_SERVICE_AGREEMENT_EXCLUDE_LATEST_TOKEN_ID; } catch (error) { if (error.message.includes(EXPECTED_TRANSACTION_ERRORS.NO_MINTED_ASSETS)) { this.logger.info( @@ -53,12 +56,22 @@ class BlockchainGetLatestServiceAgreement extends Command { ); return; } - throw error; + this.logger.error( + `Unable to process agreement data for asset contract ${contract}. Error: ${error}`, + ); + return; } const latestDbTokenId = (await this.repositoryModuleManager.getLatestServiceAgreementTokenId(blockchain)) ?? 0; + if (latestBlockchainTokenId < latestDbTokenId) { + this.logger.debug( + `Get latest service agreement: No new agreements found on blockchain: ${blockchain}.`, + ); + return; + } + this.logger.debug( `Get latest service agreement: Latest token id on chain: ${latestBlockchainTokenId}, latest token id in database: ${latestDbTokenId} on blockchain: ${blockchain}`, ); @@ -75,17 +88,15 @@ class BlockchainGetLatestServiceAgreement extends Command { ); if ( getAgreementDataPromise.length === tokenIdDifference || - getAgreementDataPromise.length === BATCH_SIZE + getAgreementDataPromise.length === GET_LATEST_SERVICE_AGREEMENT_BATCH_SIZE ) { - // eslint-disable-next-line no-await-in-loop const missingAgreements = await Promise.all(getAgreementDataPromise); - // eslint-disable-next-line no-await-in-loop await this.repositoryModuleManager.bulkCreateServiceAgreementRecords( missingAgreements.filter((agreement) => agreement != null), ); getAgreementDataPromise = []; - tokenIdDifference -= BATCH_SIZE; + tokenIdDifference -= GET_LATEST_SERVICE_AGREEMENT_BATCH_SIZE; } } if (latestBlockchainTokenId - latestDbTokenId !== 0) { @@ -103,56 +114,76 @@ class BlockchainGetLatestServiceAgreement extends Command { contract, hashFunctionId = CONTENT_ASSET_HASH_FUNCTION_ID, ) { - this.logger.debug( - `Get latest service agreement: Getting agreement data for token id: ${tokenId} on blockchain: ${blockchain}`, - ); - const assertionIds = await this.blockchainModuleManager.getAssertionIds( - blockchain, - contract, - tokenId, - ); - const keyword = await this.ualService.calculateLocationKeyword( - blockchain, - contract, - tokenId, - assertionIds[0], - ); - const agreementId = await this.serviceAgreementService.generateId( - blockchain, - contract, - tokenId, - keyword, - hashFunctionId, - ); - const agreementData = await this.blockchainModuleManager.getAgreementData( - blockchain, - agreementId, - ); + try { + this.logger.debug( + `Get latest service agreement: Getting agreement data for token id: ${tokenId} on blockchain: ${blockchain}`, + ); + let assertionIds = []; + let retryCount = 0; + + while (assertionIds.length === 0) { + if (retryCount === GET_ASSERTION_IDS_MAX_RETRY_COUNT) { + throw Error( + `Get latest service agreement: Unable to get assertion ids for token id: ${tokenId} on blockchain: ${blockchain}`, + ); + } + this.logger.debug( + `Get latest service agreement: getting assertion ids retry ${retryCount} for token id: ${tokenId} on blockchain: ${blockchain}`, + ); + assertionIds = await this.blockchainModuleManager.getAssertionIds( + blockchain, + contract, + tokenId, + ); + retryCount += 1; + await sleep(GET_ASSERTION_IDS_RETRY_DELAY_IN_SECONDS * 1000); + } - if (!agreementData) { - this.logger.warn( - `Unable to fetch agreement data while processing asset created event for agreement id: ${agreementId}, blockchain id: ${blockchain}`, + const keyword = await this.ualService.calculateLocationKeyword( + blockchain, + contract, + tokenId, + assertionIds[0], + ); + const agreementId = await this.serviceAgreementService.generateId( + blockchain, + contract, + tokenId, + keyword, + hashFunctionId, + ); + const agreementData = await this.blockchainModuleManager.getAgreementData( + blockchain, + agreementId, ); - } - const latestStateIndex = assertionIds.length - 1; - - return { - blockchainId: blockchain, - assetStorageContractAddress: contract, - tokenId, - agreementId, - startTime: agreementData.startTime, - epochsNumber: agreementData.epochsNumber, - epochLength: agreementData.epochLength, - scoreFunctionId: agreementData.scoreFunctionId, - stateIndex: latestStateIndex, - assertionId: assertionIds[latestStateIndex], - hashFunctionId, - keyword, - proofWindowOffsetPerc: agreementData.proofWindowOffsetPerc, - dataSource: SERVICE_AGREEMENT_SOURCES.NODE, - }; + if (!agreementData) { + throw Error( + `Get latest service agreement: Unable to fetch agreement data while processing asset created event for agreement id: ${agreementId}, blockchain id: ${blockchain}`, + ); + } + + const latestStateIndex = assertionIds.length - 1; + + return { + blockchainId: blockchain, + assetStorageContractAddress: contract, + tokenId, + agreementId, + startTime: agreementData.startTime, + epochsNumber: agreementData.epochsNumber, + epochLength: agreementData.epochLength, + scoreFunctionId: agreementData.scoreFunctionId, + stateIndex: latestStateIndex, + assertionId: assertionIds[latestStateIndex], + hashFunctionId, + keyword, + proofWindowOffsetPerc: agreementData.proofWindowOffsetPerc, + dataSource: SERVICE_AGREEMENT_SOURCES.NODE, + }; + } catch (error) { + this.logger.error(error.message); + } } /** diff --git a/src/constants/constants.js b/src/constants/constants.js index 66f7ad18a..65c409ff7 100644 --- a/src/constants/constants.js +++ b/src/constants/constants.js @@ -548,6 +548,10 @@ export const ARCHIVE_UPDATE_RESPONSES_FOLDER = 'update_responses'; */ export const COMMAND_QUEUE_PARALLELISM = 100; +export const GET_LATEST_SERVICE_AGREEMENT_BATCH_SIZE = 50; +export const GET_ASSERTION_IDS_MAX_RETRY_COUNT = 5; +export const GET_ASSERTION_IDS_RETRY_DELAY_IN_SECONDS = 2; +export const GET_LATEST_SERVICE_AGREEMENT_EXCLUDE_LATEST_TOKEN_ID = 10; /** * @constant {object} HTTP_API_ROUTES - * HTTP API Routes with parameters From 285ffa6b038540131a3919d98080509e057ef3ee Mon Sep 17 00:00:00 2001 From: Djordje Kovacevic Date: Fri, 19 Apr 2024 14:11:19 +0200 Subject: [PATCH 09/13] Updated logic for fetching latest token ids --- ...blockchain-get-latest-service-agreement.js | 34 +++++++++++++++---- src/constants/constants.js | 6 +++- 2 files changed, 32 insertions(+), 8 deletions(-) diff --git a/src/commands/common/get-latest-service-agreement/blockchain-get-latest-service-agreement.js b/src/commands/common/get-latest-service-agreement/blockchain-get-latest-service-agreement.js index 3ee827b17..ee4189a60 100644 --- a/src/commands/common/get-latest-service-agreement/blockchain-get-latest-service-agreement.js +++ b/src/commands/common/get-latest-service-agreement/blockchain-get-latest-service-agreement.js @@ -31,18 +31,29 @@ class BlockchainGetLatestServiceAgreement extends Command { const assetStorageContractAddresses = this.blockchainModuleManager.getAssetStorageContractAddresses(blockchain); - await Promise.all( + const results = await Promise.all( assetStorageContractAddresses.map((contract) => - this.updateAgreementDataForAssetContract(contract, blockchain), + this.updateAgreementDataForAssetContract( + contract, + blockchain, + command.data[contract], + ), ), ); + results.forEach((result) => { + if (result) { + // eslint-disable-next-line no-param-reassign + command.data[result.contract] = result.lastProcessedTokenId; + } + }); + return Command.repeat(); } - async updateAgreementDataForAssetContract(contract, blockchain) { + async updateAgreementDataForAssetContract(contract, blockchain, lastProcessedTokenId) { this.logger.info( - `Get latest service agreement: Starting get latest service agreement command for blockchain: ${blockchain}`, + `Get latest service agreement: Starting get latest service agreement command, last processed token id: ${lastProcessedTokenId} for blockchain: ${blockchain}`, ); let latestBlockchainTokenId; try { @@ -63,13 +74,18 @@ class BlockchainGetLatestServiceAgreement extends Command { } const latestDbTokenId = - (await this.repositoryModuleManager.getLatestServiceAgreementTokenId(blockchain)) ?? 0; + lastProcessedTokenId ?? + (await this.repositoryModuleManager.getLatestServiceAgreementTokenId(blockchain)) ?? + 0; if (latestBlockchainTokenId < latestDbTokenId) { this.logger.debug( `Get latest service agreement: No new agreements found on blockchain: ${blockchain}.`, ); - return; + return { + contract, + lastProcessedTokenId: latestDbTokenId, + }; } this.logger.debug( @@ -99,13 +115,17 @@ class BlockchainGetLatestServiceAgreement extends Command { tokenIdDifference -= GET_LATEST_SERVICE_AGREEMENT_BATCH_SIZE; } } - if (latestBlockchainTokenId - latestDbTokenId !== 0) { + if (latestBlockchainTokenId - latestDbTokenId > 0) { this.logger.debug( `Get latest service agreement: Successfully fetched ${ latestBlockchainTokenId - latestDbTokenId } on blockchain: ${blockchain}`, ); } + return { + contract, + lastProcessedTokenId: latestDbTokenId, + }; } async getAgreementDataForToken( diff --git a/src/constants/constants.js b/src/constants/constants.js index 65c409ff7..be2b6d2da 100644 --- a/src/constants/constants.js +++ b/src/constants/constants.js @@ -549,9 +549,13 @@ export const ARCHIVE_UPDATE_RESPONSES_FOLDER = 'update_responses'; export const COMMAND_QUEUE_PARALLELISM = 100; export const GET_LATEST_SERVICE_AGREEMENT_BATCH_SIZE = 50; + export const GET_ASSERTION_IDS_MAX_RETRY_COUNT = 5; + export const GET_ASSERTION_IDS_RETRY_DELAY_IN_SECONDS = 2; -export const GET_LATEST_SERVICE_AGREEMENT_EXCLUDE_LATEST_TOKEN_ID = 10; + +export const GET_LATEST_SERVICE_AGREEMENT_EXCLUDE_LATEST_TOKEN_ID = 1; + /** * @constant {object} HTTP_API_ROUTES - * HTTP API Routes with parameters From 122f1176f09f9bf384b81e89d836c641309d96dd Mon Sep 17 00:00:00 2001 From: Djordje Kovacevic Date: Fri, 19 Apr 2024 14:46:33 +0200 Subject: [PATCH 10/13] Updated repository to ignore duplicates on adding new service agreement --- .../sequelize/repositories/service-agreement-repository.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/modules/repository/implementation/sequelize/repositories/service-agreement-repository.js b/src/modules/repository/implementation/sequelize/repositories/service-agreement-repository.js index 562f9d061..19ffb462a 100644 --- a/src/modules/repository/implementation/sequelize/repositories/service-agreement-repository.js +++ b/src/modules/repository/implementation/sequelize/repositories/service-agreement-repository.js @@ -85,7 +85,7 @@ class ServiceAgreementRepository { async bulkCreateServiceAgreementRecords(serviceAgreements) { return this.model.bulkCreate(serviceAgreements, { - validate: true, + ignoreDuplicates: true, }); } From ec9f9dc5cd7942247a1624164d76aefb5dceac14 Mon Sep 17 00:00:00 2001 From: Djordje Kovacevic Date: Tue, 23 Apr 2024 12:53:05 +0200 Subject: [PATCH 11/13] Added additional logs, added check if agreement data already exists --- .../blockchain-get-latest-service-agreement.js | 11 ++++++++++- .../repositories/service-agreement-repository.js | 10 ++++++++++ src/modules/repository/repository-module-manager.js | 9 +++++++++ 3 files changed, 29 insertions(+), 1 deletion(-) diff --git a/src/commands/common/get-latest-service-agreement/blockchain-get-latest-service-agreement.js b/src/commands/common/get-latest-service-agreement/blockchain-get-latest-service-agreement.js index 1782490af..c95e8a65b 100644 --- a/src/commands/common/get-latest-service-agreement/blockchain-get-latest-service-agreement.js +++ b/src/commands/common/get-latest-service-agreement/blockchain-get-latest-service-agreement.js @@ -45,6 +45,9 @@ class BlockchainGetLatestServiceAgreement extends Command { if (result) { // eslint-disable-next-line no-param-reassign command.data[result.contract] = result.lastProcessedTokenId; + this.logger.debug( + `Get latest service agreement: updating last processed token id: ${result.lastProcessedTokenId} for blockchain ${blockchain}`, + ); } }); @@ -131,7 +134,7 @@ class BlockchainGetLatestServiceAgreement extends Command { } return { contract, - lastProcessedTokenId: latestDbTokenId, + lastProcessedTokenId: latestBlockchainTokenId, }; } @@ -142,6 +145,12 @@ class BlockchainGetLatestServiceAgreement extends Command { hashFunctionId = CONTENT_ASSET_HASH_FUNCTION_ID, ) { try { + if (await this.repositoryModuleManager.serviceAgreementExists(blockchain, tokenId)) { + this.logger.debug( + `Get latest service agreement: data exists in repository for token id: ${tokenId} on blockchain: ${blockchain}`, + ); + return; + } this.logger.debug( `Get latest service agreement: Getting agreement data for token id: ${tokenId} on blockchain: ${blockchain}`, ); diff --git a/src/modules/repository/implementation/sequelize/repositories/service-agreement-repository.js b/src/modules/repository/implementation/sequelize/repositories/service-agreement-repository.js index 19ffb462a..062a6876e 100644 --- a/src/modules/repository/implementation/sequelize/repositories/service-agreement-repository.js +++ b/src/modules/repository/implementation/sequelize/repositories/service-agreement-repository.js @@ -83,6 +83,16 @@ class ServiceAgreementRepository { ); } + async serviceAgreementExists(blockchainId, tokenId) { + const agreementRecord = await this.model.findOne({ + where: { + blockchainId, + tokenId, + }, + }); + return !!agreementRecord; + } + async bulkCreateServiceAgreementRecords(serviceAgreements) { return this.model.bulkCreate(serviceAgreements, { ignoreDuplicates: true, diff --git a/src/modules/repository/repository-module-manager.js b/src/modules/repository/repository-module-manager.js index e39a44b81..9aefcb553 100644 --- a/src/modules/repository/repository-module-manager.js +++ b/src/modules/repository/repository-module-manager.js @@ -344,6 +344,15 @@ class RepositoryModuleManager extends BaseModuleManager { } } + async serviceAgreementExists(blockchain, tokenId) { + if (this.initialized) { + return this.getRepository('service_agreement').serviceAgreementExists( + blockchain, + tokenId, + ); + } + } + async bulkCreateServiceAgreementRecords(records) { if (this.initialized) { return this.getRepository('service_agreement').bulkCreateServiceAgreementRecords( From a5f8fa8b2efffa36d5904c03c1065433e537c500 Mon Sep 17 00:00:00 2001 From: Djordje Kovacevic Date: Tue, 23 Apr 2024 12:56:10 +0200 Subject: [PATCH 12/13] version bump --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 1690bab2e..9272218fc 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "origintrail_node", - "version": "6.3.0+hotfix.1", + "version": "6.3.0+hotfix.2", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "origintrail_node", - "version": "6.3.0+hotfix.1", + "version": "6.3.0+hotfix.2", "license": "ISC", "dependencies": { "@comunica/query-sparql": "^2.4.3", diff --git a/package.json b/package.json index 829f67112..2d3f748c9 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "origintrail_node", - "version": "6.3.0+hotfix.1", + "version": "6.3.0+hotfix.2", "description": "OTNode V6", "main": "index.js", "type": "module", From 3c800017a053b9db840a363176f93eaf08ae8b43 Mon Sep 17 00:00:00 2001 From: Djordje Kovacevic Date: Wed, 24 Apr 2024 15:34:40 +0200 Subject: [PATCH 13/13] Version updated - removed hotfix from version --- package-lock.json | 4 ++-- package.json | 2 +- .../blockchain-get-latest-service-agreement.js | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/package-lock.json b/package-lock.json index 9272218fc..3a18c9d49 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "origintrail_node", - "version": "6.3.0+hotfix.2", + "version": "6.3.0", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "origintrail_node", - "version": "6.3.0+hotfix.2", + "version": "6.3.0", "license": "ISC", "dependencies": { "@comunica/query-sparql": "^2.4.3", diff --git a/package.json b/package.json index 2d3f748c9..57fad629e 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "origintrail_node", - "version": "6.3.0+hotfix.2", + "version": "6.3.0", "description": "OTNode V6", "main": "index.js", "type": "module", diff --git a/src/commands/common/get-latest-service-agreement/blockchain-get-latest-service-agreement.js b/src/commands/common/get-latest-service-agreement/blockchain-get-latest-service-agreement.js index c95e8a65b..26e79a6f2 100644 --- a/src/commands/common/get-latest-service-agreement/blockchain-get-latest-service-agreement.js +++ b/src/commands/common/get-latest-service-agreement/blockchain-get-latest-service-agreement.js @@ -79,7 +79,7 @@ class BlockchainGetLatestServiceAgreement extends Command { const latestDbTokenId = lastProcessedTokenId ?? (await this.repositoryModuleManager.getLatestServiceAgreementTokenId(blockchain)) ?? - 0; + latestBlockchainTokenId; if (latestBlockchainTokenId < latestDbTokenId) { this.logger.debug(