From bd8bb9a8fa5ce09f35ba2396609d02911174958c Mon Sep 17 00:00:00 2001 From: djordjekovac Date: Thu, 27 Apr 2023 11:47:09 +0200 Subject: [PATCH 01/10] Don't start the node until it processed all blockchain events (#2453) * Don't start the node untill it processed all blockchain events * Logs added * Fixed PR comments --------- Co-authored-by: zeroxbt <89495162+zeroxbt@users.noreply.github.com> --- ot-node.js | 7 +- src/constants/constants.js | 2 +- .../blockchain-event-listener-service.js | 65 ++++++++++++------- 3 files changed, 45 insertions(+), 29 deletions(-) diff --git a/ot-node.js b/ot-node.js index 93d5b1e65..04d5b0ff8 100644 --- a/ot-node.js +++ b/ot-node.js @@ -59,7 +59,7 @@ class OTNode { await this.initializeShardingTableService(); await this.initializeTelemetryInjectionService(); - this.initializeBlockchainEventListenerService(); + await this.initializeBlockchainEventListenerService(); await this.initializeCommandExecutor(); await this.initializeRouters(); @@ -140,10 +140,11 @@ class OTNode { this.logger.info('Event emitter initialized'); } - initializeBlockchainEventListenerService() { + async initializeBlockchainEventListenerService() { try { const eventListenerService = this.container.resolve('blockchainEventListenerService'); - eventListenerService.initialize(); + await eventListenerService.initialize(); + eventListenerService.startListeningOnEvents(); this.logger.info('Event Listener Service initialized successfully'); } catch (error) { this.logger.error( diff --git a/src/constants/constants.js b/src/constants/constants.js index 18976c75d..90ed9ab9f 100644 --- a/src/constants/constants.js +++ b/src/constants/constants.js @@ -25,7 +25,7 @@ export const LIBP2P_KEY_FILENAME = 'privateKey'; export const TRIPLE_STORE_CONNECT_MAX_RETRIES = 10; -export const DEFAULT_BLOCKCHAIN_EVENT_SYNC_PERIOD_IN_MILLS = 15 * 24 * 60 * 60 * 1000; +export const DEFAULT_BLOCKCHAIN_EVENT_SYNC_PERIOD_IN_MILLS = 15 * 24 * 60 * 60 * 1000; // 15 days export const MAXIMUM_NUMBERS_OF_BLOCKS_TO_FETCH = 500; diff --git a/src/service/blockchain-event-listener-service.js b/src/service/blockchain-event-listener-service.js index 2811b563d..6c7fd671c 100644 --- a/src/service/blockchain-event-listener-service.js +++ b/src/service/blockchain-event-listener-service.js @@ -27,13 +27,51 @@ class BlockchainEventListenerService { this.ualService = ctx.ualService; } - initialize() { + async initialize() { + const promises = []; + for (const blockchainId of this.blockchainModuleManager.getImplementationNames()) { + this.logger.info( + `Initializing blockchain event listener for blockchain ${blockchainId}, handling missed events`, + ); + promises.push(this.fetchAndHandleBlockchainEvents(blockchainId)); + } + await Promise.all(promises); + } + + startListeningOnEvents() { for (const blockchainId of this.blockchainModuleManager.getImplementationNames()) { this.listenOnBlockchainEvents(blockchainId); this.logger.info(`Event listener initialized for blockchain: '${blockchainId}'.`); } } + async fetchAndHandleBlockchainEvents(blockchainId) { + const devEnvironment = + process.env.NODE_ENV === NODE_ENVIRONMENTS.DEVELOPMENT || + process.env.NODE_ENV === NODE_ENVIRONMENTS.TEST; + + const currentBlock = await this.blockchainModuleManager.getBlockNumber(); + const syncContractEventsPromises = [ + this.getContractEvents(blockchainId, CONTRACTS.SHARDING_TABLE_CONTRACT, currentBlock), + this.getContractEvents(blockchainId, CONTRACTS.STAKING_CONTRACT, currentBlock), + this.getContractEvents(blockchainId, CONTRACTS.PROFILE_CONTRACT, currentBlock), + this.getContractEvents( + blockchainId, + CONTRACTS.COMMIT_MANAGER_V1_U1_CONTRACT, + currentBlock, + ), + ]; + + if (!devEnvironment) { + syncContractEventsPromises.push( + this.getContractEvents(blockchainId, CONTRACTS.HUB_CONTRACT, currentBlock), + ); + } + const contractEvents = await Promise.all(syncContractEventsPromises); + + await this.handleBlockchainEvents(contractEvents.flatMap((events) => events)); + } + listenOnBlockchainEvents(blockchainId) { const devEnvironment = process.env.NODE_ENV === NODE_ENVIRONMENTS.DEVELOPMENT || @@ -49,30 +87,7 @@ class BlockchainEventListenerService { if (working) return; try { working = true; - const currentBlock = await this.blockchainModuleManager.getBlockNumber(); - const syncContractEventsPromises = [ - this.getContractEvents( - blockchainId, - CONTRACTS.SHARDING_TABLE_CONTRACT, - currentBlock, - ), - this.getContractEvents(blockchainId, CONTRACTS.STAKING_CONTRACT, currentBlock), - this.getContractEvents(blockchainId, CONTRACTS.PROFILE_CONTRACT, currentBlock), - this.getContractEvents( - blockchainId, - CONTRACTS.COMMIT_MANAGER_V1_U1_CONTRACT, - currentBlock, - ), - ]; - - if (!devEnvironment) { - syncContractEventsPromises.push( - this.getContractEvents(blockchainId, CONTRACTS.HUB_CONTRACT, currentBlock), - ); - } - const contractEvents = await Promise.all(syncContractEventsPromises); - - await this.handleBlockchainEvents(contractEvents.flatMap((events) => events)); + await this.fetchAndHandleBlockchainEvents(blockchainId); fetchEventsFailedCount[blockchainId] = 0; } catch (e) { if (fetchEventsFailedCount[blockchainId] >= MAXIMUM_FETCH_EVENTS_FAILED_COUNT) { From bd23d94f83877a316fa9107ed87eeacff4fc4bb4 Mon Sep 17 00:00:00 2001 From: zeroxbt <89495162+zeroxbt@users.noreply.github.com> Date: Thu, 27 Apr 2023 12:26:07 +0200 Subject: [PATCH 02/10] ethers wait for transaction receipt (#2457) * ethers wait for transaction receipt * handle timeouts * Increased timeout for blockchain start * Removed caching of compiled contracts --------- Co-authored-by: Djordje Kovacevic --- .github/workflows/TEST-bdd.yml | 6 ------ scripts/set-ask.js | 8 ++++++-- scripts/set-operator-fee.js | 8 ++++++-- scripts/set-stake.js | 11 ++++++++--- src/constants/constants.js | 2 +- src/modules/blockchain/implementation/web3-service.js | 10 ++++++++-- test/bdd/steps/blockchain.mjs | 2 +- 7 files changed, 30 insertions(+), 17 deletions(-) diff --git a/.github/workflows/TEST-bdd.yml b/.github/workflows/TEST-bdd.yml index 495ec8138..7fb92f8fe 100644 --- a/.github/workflows/TEST-bdd.yml +++ b/.github/workflows/TEST-bdd.yml @@ -56,12 +56,6 @@ jobs: - run: mkdir -p $CUCUMBER_ARTIFACTS_DIR - run: sudo chmod -R 777 $CUCUMBER_ARTIFACTS_DIR - run: npm explore dkg-evm-module -- npm run compile; - - name: Cache compiled contracts - uses: actions/cache@v2 - with: - path: node_modules/dkg-evm-module/artifacts - key: ${{ runner.os }}-contracts-${{ hashFiles('**/artifacts/contracts') }} - restore-keys: ${{ runner.os }}-contracts- - run: npm run test:bdd; - uses: actions/upload-artifact@v2 if: ${{ always() }} diff --git a/scripts/set-ask.js b/scripts/set-ask.js index 43b9a6b44..7b710e56c 100644 --- a/scripts/set-ask.js +++ b/scripts/set-ask.js @@ -1,7 +1,10 @@ /* eslint-disable no-console */ import { ethers } from 'ethers'; import { createRequire } from 'module'; -import { NODE_ENVIRONMENTS } from '../src/constants/constants.js'; +import { + NODE_ENVIRONMENTS, + TRANSACTION_POLLING_TIMEOUT_MILLIS, +} from '../src/constants/constants.js'; import validateArguments from './utils.js'; const require = createRequire(import.meta.url); @@ -32,10 +35,11 @@ async function setAsk(rpcEndpoint, ask, walletPrivateKey, hubContractAddress) { const askWei = ethers.utils.parseEther(ask); - await profile.setAsk(identityId, askWei, { + const tx = await profile.setAsk(identityId, askWei, { gasPrice: devEnvironment ? undefined : 8, gasLimit: 500_000, }); + await provider.waitForTransaction(tx.hash, null, TRANSACTION_POLLING_TIMEOUT_MILLIS); } const expectedArguments = ['rpcEndpoint', 'ask', 'privateKey', 'hubContractAddress']; diff --git a/scripts/set-operator-fee.js b/scripts/set-operator-fee.js index 66421c53c..876c7ff33 100644 --- a/scripts/set-operator-fee.js +++ b/scripts/set-operator-fee.js @@ -1,7 +1,10 @@ /* eslint-disable no-console */ import { ethers } from 'ethers'; import { createRequire } from 'module'; -import { NODE_ENVIRONMENTS } from '../src/constants/constants.js'; +import { + NODE_ENVIRONMENTS, + TRANSACTION_POLLING_TIMEOUT_MILLIS, +} from '../src/constants/constants.js'; import validateArguments from './utils.js'; const require = createRequire(import.meta.url); @@ -30,10 +33,11 @@ async function setOperatorFee(rpcEndpoint, operatorFee, walletPrivateKey, hubCon const identityId = await identityStorage.getIdentityId(wallet.address); - stakingContract.setOperatorFee(identityId, operatorFee, { + const tx = await stakingContract.setOperatorFee(identityId, operatorFee, { gasPrice: process.env.NODE_ENV === NODE_ENVIRONMENTS.DEVELOPMENT ? undefined : 8, gasLimit: 500_000, }); + await provider.waitForTransaction(tx.hash, null, TRANSACTION_POLLING_TIMEOUT_MILLIS); } const expectedArguments = ['rpcEndpoint', 'operatorFee', 'privateKey', 'hubContractAddress']; diff --git a/scripts/set-stake.js b/scripts/set-stake.js index 1852b754b..5b6ef019b 100644 --- a/scripts/set-stake.js +++ b/scripts/set-stake.js @@ -1,7 +1,10 @@ /* eslint-disable no-console */ import { ethers } from 'ethers'; import { createRequire } from 'module'; -import { NODE_ENVIRONMENTS } from '../src/constants/constants.js'; +import { + NODE_ENVIRONMENTS, + TRANSACTION_POLLING_TIMEOUT_MILLIS, +} from '../src/constants/constants.js'; import validateArguments from './utils.js'; const require = createRequire(import.meta.url); @@ -48,15 +51,17 @@ async function setStake( const stakeWei = ethers.utils.parseEther(stake); - await tokenContract.increaseAllowance(stakingContractAddress, stakeWei, { + let tx = await tokenContract.increaseAllowance(stakingContractAddress, stakeWei, { gasPrice: devEnvironment ? undefined : 8, gasLimit: 500_000, }); + await provider.waitForTransaction(tx.hash, null, TRANSACTION_POLLING_TIMEOUT_MILLIS); // TODO: Add ABI instead of hard-coded function definition - await stakingContract['addStake(uint72,uint96)'](identityId, stakeWei, { + tx = await stakingContract['addStake(uint72,uint96)'](identityId, stakeWei, { gasPrice: devEnvironment ? undefined : 1_000, gasLimit: 500_000, }); + await provider.waitForTransaction(tx.hash, null, TRANSACTION_POLLING_TIMEOUT_MILLIS); } const expectedArguments = [ diff --git a/src/constants/constants.js b/src/constants/constants.js index 90ed9ab9f..3037d35ce 100644 --- a/src/constants/constants.js +++ b/src/constants/constants.js @@ -17,7 +17,7 @@ export const COMMIT_BLOCK_DURATION_IN_BLOCKS = 5; export const BLOCK_TIME = 12; export const COMMITS_DELAY_BETWEEN_NODES_IN_BLOCKS = 2; -export const TRANSACTION_POLLING_TIMEOUT = 50; +export const TRANSACTION_POLLING_TIMEOUT_MILLIS = 50 * 1000; export const LIBP2P_KEY_DIRECTORY = 'libp2p'; diff --git a/src/modules/blockchain/implementation/web3-service.js b/src/modules/blockchain/implementation/web3-service.js index ae8ab4212..ad0248095 100644 --- a/src/modules/blockchain/implementation/web3-service.js +++ b/src/modules/blockchain/implementation/web3-service.js @@ -9,6 +9,7 @@ import { MAXIMUM_NUMBERS_OF_BLOCKS_TO_FETCH, TRANSACTION_QUEUE_CONCURRENCY, FIXED_GAS_LIMIT_METHODS, + TRANSACTION_POLLING_TIMEOUT_MILLIS, } from '../../../constants/constants.js'; const require = createRequire(import.meta.url); @@ -351,17 +352,22 @@ class Web3Service { 'Sending signed transaction to blockchain, calling method: ' + `${functionName} with gas limit: ${gas.toString()} and gasPrice ${gasPrice.toString()}`, ); - result = await contractInstance[functionName](...args, { + const tx = await contractInstance[functionName](...args, { gasPrice, gasLimit: gas, }); + result = await this.provider.waitForTransaction( + tx.hash, + null, + TRANSACTION_POLLING_TIMEOUT_MILLIS, + ); } catch (error) { this.logger.warn( `Failed executing smart contract function ${functionName}. Error: ${error.message}`, ); if ( !transactionRetried && - (error.message.includes(`Transaction was not mined within`) || + (error.message.includes(`timeout exceeded`) || error.message.includes(`Pool(TooLowPriority`)) ) { gasPrice = Math.ceil(gasPrice * 1.2); diff --git a/test/bdd/steps/blockchain.mjs b/test/bdd/steps/blockchain.mjs index e732dd304..16cac4c17 100644 --- a/test/bdd/steps/blockchain.mjs +++ b/test/bdd/steps/blockchain.mjs @@ -3,7 +3,7 @@ import { expect } from 'chai'; import fs from 'fs'; import LocalBlockchain from './lib/local-blockchain.mjs'; -Given(/^the blockchain is set up$/, { timeout: 180_000 }, function blockchainSetup(done) { +Given(/^the blockchain is set up$/, { timeout: 240_000 }, function blockchainSetup(done) { this.logger.log('Starting blockchain'); expect(this.state.localBlockchain, "localBlockchain shouldn't be defined").to.be.equal(null); const blockchainConsole = new console.Console( From e1550c3808fa6dea8d66a42052c97586da6bc829 Mon Sep 17 00:00:00 2001 From: zeroxbt <89495162+zeroxbt@users.noreply.github.com> Date: Tue, 9 May 2023 14:59:08 +0200 Subject: [PATCH 03/10] add migrations for improved epoch check (#2451) * add repository migrations and models * add migration file * add first implementation of service agreement migration * add execute migration at start * fix typo * small fixes * update migration name and order of execution * update service agreements table on req and event * update commit and proof table on succesful txs * Updated naming of new tables * Added epoch check command * save last processed token id in file * bug fixes, better error handling and logs * update commit and proof table on failed tx * remove unnecessary tables * update service agreement table * add last commit epoch and last proof epoch in migration * fix primary key too long * update migration and model * bump version * add remaining assets to process log * add concurrency to migration, remove fileservice logs * remove new epoch check command * handle service agreement events * fix service agreement extended event handler --------- Co-authored-by: Djordje Kovacevic --- ot-node.js | 32 +++- package-lock.json | 4 +- package.json | 2 +- .../publish/receiver/submit-commit-command.js | 9 + .../publish/receiver/submit-proofs-command.js | 2 + .../v1-0-0-handle-store-request-command.js | 22 +++ .../v1-0-0-handle-update-request-command.js | 2 + src/constants/constants.js | 5 + .../service-agreements-metadata-migration.js | 172 ++++++++++++++++++ ...0230419140000-create-service-agreements.js | 66 +++++++ .../sequelize/models/service-agreement.js | 70 +++++++ .../sequelize/sequelize-repository.js | 64 ++++++- .../repository/repository-module-manager.js | 69 +++++++ .../blockchain-event-listener-service.js | 67 ++++++- src/service/file-service.js | 6 +- 15 files changed, 577 insertions(+), 15 deletions(-) create mode 100644 src/migration/service-agreements-metadata-migration.js create mode 100644 src/modules/repository/implementation/sequelize/migrations/20230419140000-create-service-agreements.js create mode 100644 src/modules/repository/implementation/sequelize/models/service-agreement.js diff --git a/ot-node.js b/ot-node.js index 04d5b0ff8..a11a1c998 100644 --- a/ot-node.js +++ b/ot-node.js @@ -12,6 +12,7 @@ import OtAutoUpdater from './src/modules/auto-updater/implementation/ot-auto-upd import PullBlockchainShardingTableMigration from './src/migration/pull-sharding-table-migration.js'; import TripleStoreUserConfigurationMigration from './src/migration/triple-store-user-configuration-migration.js'; import PrivateAssetsMetadataMigration from './src/migration/private-assets-metadata-migration.js'; +import ServiceAgreementsMetadataMigration from './src/migration/service-agreements-metadata-migration.js'; import RemoveAgreementStartEndTimeMigration from './src/migration/remove-agreement-start-end-time-migration.js'; import MarkOldBlockchainEventsAsProcessedMigration from './src/migration/mark-old-blockchain-events-as-processed-migration.js'; import TripleStoreMetadataMigration from './src/migration/triple-store-metadata-migration.js'; @@ -53,7 +54,8 @@ class OTNode { await this.executePrivateAssetsMetadataMigration(); await this.executeRemoveAgreementStartEndTimeMigration(); await this.executeMarkOldBlockchainEventsAsProcessedMigration(); - this.executeTripleStoreMetadataMigration(); + await this.executeTripleStoreMetadataMigration(); + this.executeServiceAgreementsMetadataMigration(); await this.createProfiles(); @@ -321,6 +323,34 @@ class OTNode { } } + async executeServiceAgreementsMetadataMigration() { + if ( + process.env.NODE_ENV === NODE_ENVIRONMENTS.DEVELOPMENT || + process.env.NODE_ENV === NODE_ENVIRONMENTS.TEST + ) + return; + + const blockchainModuleManager = this.container.resolve('blockchainModuleManager'); + const repositoryModuleManager = this.container.resolve('repositoryModuleManager'); + const tripleStoreService = this.container.resolve('tripleStoreService'); + const serviceAgreementService = this.container.resolve('serviceAgreementService'); + const ualService = this.container.resolve('ualService'); + + const migration = new ServiceAgreementsMetadataMigration( + 'serviceAgreementsMetadataMigration', + this.logger, + this.config, + tripleStoreService, + blockchainModuleManager, + repositoryModuleManager, + serviceAgreementService, + ualService, + ); + if (!(await migration.migrationAlreadyExecuted())) { + await migration.migrate(); + } + } + async executeRemoveAgreementStartEndTimeMigration() { if ( process.env.NODE_ENV === NODE_ENVIRONMENTS.DEVELOPMENT || diff --git a/package-lock.json b/package-lock.json index 75c9f367f..77e0d19c2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "origintrail_node", - "version": "6.0.7", + "version": "6.0.8", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "origintrail_node", - "version": "6.0.7", + "version": "6.0.8", "license": "ISC", "dependencies": { "@comunica/query-sparql": "^2.4.3", diff --git a/package.json b/package.json index 77d3bb55e..808d24821 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "origintrail_node", - "version": "6.0.7", + "version": "6.0.8", "description": "OTNode V6", "main": "index.js", "type": "module", diff --git a/src/commands/protocols/publish/receiver/submit-commit-command.js b/src/commands/protocols/publish/receiver/submit-commit-command.js index b8650bc3c..5e436e9ce 100644 --- a/src/commands/protocols/publish/receiver/submit-commit-command.js +++ b/src/commands/protocols/publish/receiver/submit-commit-command.js @@ -15,6 +15,7 @@ class SubmitCommitCommand extends EpochCommand { this.operationIdService = ctx.operationIdService; this.shardingTableService = ctx.shardingTableService; this.networkModuleManager = ctx.networkModuleManager; + this.repositoryModuleManager = ctx.repositoryModuleManager; this.errorType = ERROR_TYPE.COMMIT_PROOF.SUBMIT_COMMIT_ERROR; } @@ -115,6 +116,14 @@ class SubmitCommitCommand extends EpochCommand { stateIndex, async (result) => { if (!result.error) { + that.logger.trace( + `Successfully executed ${command.name} for agreement id: ${agreementId} ` + + `contract: ${contract}, token id: ${tokenId}, keyword: ${keyword}, ` + + `hash function id: ${hashFunctionId}. Retry number ${ + COMMAND_RETRIES.SUBMIT_COMMIT - command.retries + 1 + }`, + ); + const currentEpochStartTime = Number(agreementData.startTime) + Number(agreementData.epochLength) * epoch; diff --git a/src/commands/protocols/publish/receiver/submit-proofs-command.js b/src/commands/protocols/publish/receiver/submit-proofs-command.js index 03bd9f2dd..817bd1415 100644 --- a/src/commands/protocols/publish/receiver/submit-proofs-command.js +++ b/src/commands/protocols/publish/receiver/submit-proofs-command.js @@ -12,6 +12,7 @@ class SubmitProofsCommand extends EpochCommand { this.blockchainModuleManager = ctx.blockchainModuleManager; this.operationIdService = ctx.operationIdService; + this.repositoryModuleManager = ctx.repositoryModuleManager; this.errorType = ERROR_TYPE.COMMIT_PROOF.SUBMIT_PROOFS_ERROR; } @@ -94,6 +95,7 @@ class SubmitProofsCommand extends EpochCommand { COMMAND_RETRIES.SUBMIT_PROOFS - command.retries + 1 }`, ); + that.operationIdService.emitChangeEvent( OPERATION_ID_STATUS.COMMIT_PROOF.SUBMIT_PROOFS_END, operationId, diff --git a/src/commands/protocols/publish/receiver/v1.0.0/v1-0-0-handle-store-request-command.js b/src/commands/protocols/publish/receiver/v1.0.0/v1-0-0-handle-store-request-command.js index 30fcc613f..e54767f90 100644 --- a/src/commands/protocols/publish/receiver/v1.0.0/v1-0-0-handle-store-request-command.js +++ b/src/commands/protocols/publish/receiver/v1.0.0/v1-0-0-handle-store-request-command.js @@ -39,6 +39,12 @@ class HandleStoreRequestCommand extends HandleProtocolMessageCommand { operationId, OPERATION_ID_STATUS.PUBLISH.VALIDATING_PUBLISH_ASSERTION_REMOTE_START, ); + const assertionIds = await this.blockchainModuleManager.getAssertionIds( + blockchain, + contract, + tokenId, + ); + const stateIndex = assertionIds.length - 1; const { assertion } = await this.operationIdService.getCachedOperationIdData(operationId); await this.validationService.validateAssertion(assertionId, blockchain, assertion); @@ -69,6 +75,22 @@ class HandleStoreRequestCommand extends HandleProtocolMessageCommand { keyword, ); + await this.repositoryModuleManager.updateServiceAgreementRecord( + blockchain, + contract, + tokenId, + agreementId, + agreementData.startTime, + agreementData.epochsNumber, + agreementData.epochLength, + agreementData.scoreFunctionId, + agreementData.proofWindowOffsetPerc, + hashFunctionId, + keyword, + assertionId, + stateIndex, + ); + await this.operationIdService.updateOperationIdStatus( operationId, OPERATION_ID_STATUS.PUBLISH.PUBLISH_LOCAL_STORE_END, diff --git a/src/commands/protocols/update/receiver/v1.0.0/v1-0-0-handle-update-request-command.js b/src/commands/protocols/update/receiver/v1.0.0/v1-0-0-handle-update-request-command.js index 88d2e3f7e..516d173e8 100644 --- a/src/commands/protocols/update/receiver/v1.0.0/v1-0-0-handle-update-request-command.js +++ b/src/commands/protocols/update/receiver/v1.0.0/v1-0-0-handle-update-request-command.js @@ -53,6 +53,8 @@ class HandleUpdateRequestCommand extends HandleProtocolMessageCommand { public: { assertion, }, + agreementId, + agreementData, }, operationId, ); diff --git a/src/constants/constants.js b/src/constants/constants.js index 3037d35ce..ee3cdfabc 100644 --- a/src/constants/constants.js +++ b/src/constants/constants.js @@ -404,6 +404,7 @@ export const CONTRACTS = { PROFILE_CONTRACT: 'ProfileContract', HUB_CONTRACT: 'HubContract', COMMIT_MANAGER_V1_U1_CONTRACT: 'CommitManagerV1U1Contract', + SERVICE_AGREEMENT_V1_CONTRACT: 'ServiceAgreementV1Contract', }; export const CONTRACT_EVENTS = { @@ -427,6 +428,10 @@ export const CONTRACT_EVENTS = { COMMIT_MANAGER_V1: { STATE_FINALIZED: 'StateFinalized', }, + SERVICE_AGREEMENT_V1: { + SERVICE_AGREEMENT_V1_EXTENDED: 'ServiceAgreementV1Extended', + SERVICE_AGREEMENT_V1_TERMINATED: 'ServiceAgreementV1Terminated', + }, }; export const NODE_ENVIRONMENTS = { diff --git a/src/migration/service-agreements-metadata-migration.js b/src/migration/service-agreements-metadata-migration.js new file mode 100644 index 000000000..33e76f4a3 --- /dev/null +++ b/src/migration/service-agreements-metadata-migration.js @@ -0,0 +1,172 @@ +/* eslint-disable no-await-in-loop */ +import path from 'path'; +import BaseMigration from './base-migration.js'; +import { + CONTENT_ASSET_HASH_FUNCTION_ID, + SCHEMA_CONTEXT, + TRIPLE_STORE_REPOSITORIES, +} from '../constants/constants.js'; + +class ServiceAgreementsMetadataMigration extends BaseMigration { + constructor( + migrationName, + logger, + config, + tripleStoreService, + blockchainModuleManager, + repositoryModuleManager, + serviceAgreementService, + ualService, + ) { + super(migrationName, logger, config); + this.blockchainModuleManager = blockchainModuleManager; + this.repositoryModuleManager = repositoryModuleManager; + this.serviceAgreementService = serviceAgreementService; + this.ualService = ualService; + this.tripleStoreService = tripleStoreService; + } + + async executeMigration() { + const migrationFolderPath = this.fileService.getMigrationFolderPath(); + const migrationInfoFileName = `${this.migrationName}_info`; + const migrationInfoPath = path.join(migrationFolderPath, migrationInfoFileName); + let migrationInfo; + if (await this.fileService.fileExists(migrationInfoPath)) { + migrationInfo = await this.fileService + ._readFile(migrationInfoPath, true) + .catch(() => {}); + } + if (!migrationInfo?.lastProcessedTokenId) { + migrationInfo = { + lastProcessedTokenId: -1, + }; + } + // get metadata of all stored assets in public current triple store + const query = `PREFIX schema: <${SCHEMA_CONTEXT}> + SELECT DISTINCT ?ual WHERE { + GRAPH { + ?ual schema:tokenId ?tokenId + } + FILTER (xsd:integer(?tokenId) > ${migrationInfo.lastProcessedTokenId}) + } + ORDER BY ASC(xsd:integer(?tokenId))`; + const assetsMetadata = await this.tripleStoreService.select( + TRIPLE_STORE_REPOSITORIES.PUBLIC_CURRENT, + query, + ); + const identities = {}; + const concurrency = 10; + let promises = []; + let assetsToProcess = assetsMetadata.length; + for (const { ual } of assetsMetadata) { + const { blockchain, contract, tokenId } = this.ualService.resolveUAL(ual); + if (!identities[blockchain]) { + identities[blockchain] = await this.blockchainModuleManager.getIdentityId( + blockchain, + ); + } + promises.push( + this.processAsset(ual, blockchain, contract, tokenId, identities[blockchain]), + ); + assetsToProcess -= 1; + if (promises.length >= concurrency) { + // eslint-disable-next-line no-await-in-loop + await Promise.all(promises); + promises = []; + await this.fileService.writeContentsToFile( + migrationFolderPath, + migrationInfoFileName, + JSON.stringify({ lastProcessedTokenId: tokenId }), + false, + ); + this.logger.trace( + `${this.migrationName} remaining assets to process: ${assetsToProcess}.`, + ); + } + } + await Promise.all(promises); + } + + async processAsset(ual, blockchain, contract, tokenId, identityId) { + // get assertion ids + const assertionIds = await this.blockchainModuleManager + .getAssertionIds(blockchain, contract, tokenId) + .catch(() => {}); + + if (!assertionIds?.length) { + this.logger.warn(`Unable to find assertion ids for asset with ual: ${ual}`); + return; + } + const stateIndex = assertionIds.length - 1; + const assertionId = assertionIds[stateIndex]; + + // calculate keyword + const keyword = this.blockchainModuleManager.encodePacked( + blockchain, + ['address', 'bytes32'], + [contract, assertionIds[0]], + ); + + // generate agreement id + const agreementId = await this.serviceAgreementService.generateId( + blockchain, + contract, + tokenId, + keyword, + CONTENT_ASSET_HASH_FUNCTION_ID, + ); + + // get agreement data + const agreementData = await this.blockchainModuleManager.getAgreementData( + blockchain, + agreementId, + ); + + // calculate current epoch + const now = await this.blockchainModuleManager.getBlockchainTimestamp(blockchain); + const epoch = Math.floor((now - agreementData.startTime) / agreementData.epochLength); + + // service agreement expired, don't update commits and proofs + if (epoch >= Number(agreementData.epochsNumber)) return; + + // get top commits + const commits = await this.blockchainModuleManager.getTopCommitSubmissions( + blockchain, + agreementId, + epoch, + stateIndex, + ); + let lastCommitEpoch = null; + let lastProofEpoch = null; + + for (const commit of commits) { + if (Number(commit.identityId) === identityId) { + lastCommitEpoch = epoch; + if (Number(commit.score) === 0) { + lastProofEpoch = epoch; + } + } + } + + // store in service_agreement table + await this.repositoryModuleManager.updateServiceAgreementRecord( + blockchain, + contract, + tokenId, + agreementId, + agreementData.startTime, + agreementData.epochsNumber, + agreementData.epochLength, + agreementData.scoreFunctionId, + agreementData.proofWindowOffsetPerc, + CONTENT_ASSET_HASH_FUNCTION_ID, + keyword, + assertionId, + stateIndex, + lastCommitEpoch, + lastProofEpoch, + ); + } +} + +export default ServiceAgreementsMetadataMigration; diff --git a/src/modules/repository/implementation/sequelize/migrations/20230419140000-create-service-agreements.js b/src/modules/repository/implementation/sequelize/migrations/20230419140000-create-service-agreements.js new file mode 100644 index 000000000..2580a2b54 --- /dev/null +++ b/src/modules/repository/implementation/sequelize/migrations/20230419140000-create-service-agreements.js @@ -0,0 +1,66 @@ +export const up = async ({ context: { queryInterface, Sequelize } }) => { + await queryInterface.createTable('service_agreement', { + blockchain_id: { + type: Sequelize.STRING, + allowNull: false, + }, + asset_storage_contract_address: { + type: Sequelize.STRING(42), + allowNull: false, + }, + token_id: { + type: Sequelize.INTEGER.UNSIGNED, + allowNull: false, + }, + agreement_id: { + type: Sequelize.STRING, + primaryKey: true, + }, + start_time: { + type: Sequelize.INTEGER.UNSIGNED, + allowNull: false, + }, + epochs_number: { + type: Sequelize.SMALLINT.UNSIGNED, + allowNull: false, + }, + epoch_length: { + type: Sequelize.INTEGER.UNSIGNED, + allowNull: false, + }, + score_function_id: { + type: Sequelize.TINYINT.UNSIGNED, + allowNull: false, + }, + state_index: { + type: Sequelize.SMALLINT.UNSIGNED, + allowNull: false, + }, + assertion_id: { + type: Sequelize.STRING, + allowNull: false, + }, + hash_function_id: { + type: Sequelize.TINYINT.UNSIGNED, + allowNull: false, + }, + keyword: { + type: Sequelize.STRING, + allowNull: false, + }, + proof_window_offset_perc: { + type: Sequelize.TINYINT.UNSIGNED, + allowNull: false, + }, + last_commit_epoch: { + type: Sequelize.SMALLINT.UNSIGNED, + }, + last_proof_epoch: { + type: Sequelize.SMALLINT.UNSIGNED, + }, + }); +}; + +export const down = async ({ context: { queryInterface } }) => { + await queryInterface.dropTable('service_agreement'); +}; diff --git a/src/modules/repository/implementation/sequelize/models/service-agreement.js b/src/modules/repository/implementation/sequelize/models/service-agreement.js new file mode 100644 index 000000000..52f6707c7 --- /dev/null +++ b/src/modules/repository/implementation/sequelize/models/service-agreement.js @@ -0,0 +1,70 @@ +export default (sequelize, DataTypes) => { + const serviceAgreement = sequelize.define( + 'service_agreement', + { + blockchain_id: { + type: DataTypes.STRING, + allowNull: false, + }, + asset_storage_contract_address: { + type: DataTypes.STRING(42), + allowNull: false, + }, + token_id: { + type: DataTypes.INTEGER.UNSIGNED, + allowNull: false, + }, + agreement_id: { + type: DataTypes.STRING, + primaryKey: true, + }, + start_time: { + type: DataTypes.INTEGER.UNSIGNED, + allowNull: false, + }, + epochs_number: { + type: DataTypes.SMALLINT.UNSIGNED, + allowNull: false, + }, + epoch_length: { + type: DataTypes.INTEGER.UNSIGNED, + allowNull: false, + }, + score_function_id: { + type: DataTypes.TINYINT.UNSIGNED, + allowNull: false, + }, + state_index: { + type: DataTypes.SMALLINT.UNSIGNED, + allowNull: false, + }, + assertion_id: { + type: DataTypes.STRING, + allowNull: false, + }, + hash_function_id: { + type: DataTypes.TINYINT.UNSIGNED, + allowNull: false, + }, + keyword: { + type: DataTypes.STRING, + allowNull: false, + }, + proof_window_offset_perc: { + type: DataTypes.TINYINT.UNSIGNED, + allowNull: false, + }, + last_commit_epoch: { + type: DataTypes.SMALLINT.UNSIGNED, + }, + last_proof_epoch: { + type: DataTypes.SMALLINT.UNSIGNED, + }, + }, + {}, + ); + serviceAgreement.associate = () => { + // associations can be defined here + }; + return serviceAgreement; +}; diff --git a/src/modules/repository/implementation/sequelize/sequelize-repository.js b/src/modules/repository/implementation/sequelize/sequelize-repository.js index c4de8506e..a61cae5ad 100644 --- a/src/modules/repository/implementation/sequelize/sequelize-repository.js +++ b/src/modules/repository/implementation/sequelize/sequelize-repository.js @@ -99,6 +99,52 @@ class SequelizeRepository { return this.models.sequelize.transaction(async (t) => execFn(t)); } + async updateServiceAgreementRecord( + blockchainId, + contract, + tokenId, + agreementId, + startTime, + epochsNumber, + epochLength, + scoreFunctionId, + proofWindowOffsetPerc, + hashFunctionId, + keyword, + assertionId, + stateIndex, + lastCommitEpoch, + lastProofEpoch, + ) { + return this.models.service_agreement.upsert({ + blockchain_id: blockchainId, + asset_storage_contract_address: contract, + token_id: tokenId, + agreement_id: agreementId, + start_time: startTime, + epochs_number: epochsNumber, + epoch_length: epochLength, + score_function_id: scoreFunctionId, + proof_window_offset_perc: proofWindowOffsetPerc, + last_commit_epoch: lastCommitEpoch, + last_proof_epoch: lastProofEpoch, + hash_function_id: hashFunctionId, + keyword, + assertion_id: assertionId, + state_index: stateIndex, + }); + } + + async removeServiceAgreementRecord(blockchainId, contract, tokenId) { + await this.models.service_agreement.destroy({ + where: { + blockchain_id: blockchainId, + asset_storage_contract_address: contract, + token_id: tokenId, + }, + }); + } + // COMMAND async updateCommand(update, opts) { await this.models.commands.update(update, opts); @@ -538,11 +584,21 @@ class SequelizeRepository { ); } - async removeBlockchainEvents(contractName) { - return this.models.blockchain_event.destroy({ - where: { - contract: contractName, + // eslint-disable-next-line no-empty-function + async getEligibleSubmitCommits() {} + + async updateServiceAgreementEpochsNumber(agreementId, epochsNumber) { + return this.models.service_agreement.update( + { epochs_number: epochsNumber }, + { + where: { agreement_id: agreementId }, }, + ); + } + + async removeServiceAgreements(agreementIds) { + return this.models.service_agreement.destroy({ + where: { agreement_id: { [Sequelize.Op.in]: agreementIds } }, }); } } diff --git a/src/modules/repository/repository-module-manager.js b/src/modules/repository/repository-module-manager.js index 156022a15..5c332a84b 100644 --- a/src/modules/repository/repository-module-manager.js +++ b/src/modules/repository/repository-module-manager.js @@ -17,6 +17,54 @@ class RepositoryModuleManager extends BaseModuleManager { } } + async updateServiceAgreementRecord( + blockchainId, + contract, + tokenId, + agreementId, + startTime, + epochsNumber, + epochLength, + scoreFunctionId, + proofWindowOffsetPerc, + hashFunctionId, + keyword, + assertionId, + stateIndex, + lastCommitEpoch, + lastProofEpoch, + ) { + if (this.initialized) { + return this.getImplementation().module.updateServiceAgreementRecord( + blockchainId, + contract, + tokenId, + agreementId, + startTime, + epochsNumber, + epochLength, + scoreFunctionId, + proofWindowOffsetPerc, + hashFunctionId, + keyword, + assertionId, + stateIndex, + lastCommitEpoch, + lastProofEpoch, + ); + } + } + + async removeServiceAgreementRecord(blockchainId, contract, tokenId) { + if (this.initialized) { + return this.getImplementation().module.removeServiceAgreementRecord( + blockchainId, + contract, + tokenId, + ); + } + } + // COMMANDS async updateCommand(update, opts) { if (this.initialized) { @@ -337,6 +385,27 @@ class RepositoryModuleManager extends BaseModuleManager { ); } } + + async getEligibleSubmitCommits() { + if (this.initialized) { + return this.getImplementation().module.getEligibleSubmitCommits(); + } + } + + async removeServiceAgreements(agreementIds) { + if (this.initialized) { + return this.getImplementation().module.removeServiceAgreements(agreementIds); + } + } + + async updateServiceAgreementEpochsNumber(agreementId, epochsNumber) { + if (this.initialized) { + return this.getImplementation().module.updateServiceAgreementEpochsNumber( + agreementId, + epochsNumber, + ); + } + } } export default RepositoryModuleManager; diff --git a/src/service/blockchain-event-listener-service.js b/src/service/blockchain-event-listener-service.js index 6c7fd671c..0cb633eb2 100644 --- a/src/service/blockchain-event-listener-service.js +++ b/src/service/blockchain-event-listener-service.js @@ -60,6 +60,11 @@ class BlockchainEventListenerService { CONTRACTS.COMMIT_MANAGER_V1_U1_CONTRACT, currentBlock, ), + this.getContractEvents( + blockchainId, + CONTRACTS.SERVICE_AGREEMENT_V1_CONTRACT, + currentBlock, + ), ]; if (!devEnvironment) { @@ -313,10 +318,7 @@ class BlockchainEventListenerService { blockchain_id: event.blockchain_id, stake: this.blockchainModuleManager.convertFromWei( event.blockchain_id, - await this.blockchainModuleManager.getNodeStake( - event.blockchain_id, - eventData.identityId, - ), + eventData.newStake, ), }; }), @@ -355,12 +357,36 @@ class BlockchainEventListenerService { await this.repositoryModuleManager.updatePeersAsk(peerRecords); } + async handleServiceAgreementV1ExtendedEvents(blockEvents) { + await Promise.all( + blockEvents.map(async (event) => { + const { agreementId } = JSON.parse(event.data); + + const { epochsNumber } = await this.blockchainModuleManager.getAgreementData( + event.blockchain_id, + agreementId, + ); + + return this.repositoryModuleManager.updateServiceAgreementEpochsNumber( + agreementId, + epochsNumber, + ); + }), + ); + } + + async handleServiceAgreementV1TerminatedEvents(blockEvents) { + await this.repositoryModuleManager.removeServiceAgreements( + blockEvents.map((event) => JSON.parse(event.data).agreementId), + ); + } + async handleStateFinalizedEvents(blockEvents) { // todo: find a way to safely parallelize this for (const event of blockEvents) { const eventData = JSON.parse(event.data); - const { tokenId, keyword, state } = eventData; + const { tokenId, keyword, state, stateIndex } = eventData; const blockchain = event.blockchain_id; const contract = eventData.assetContract; this.logger.trace( @@ -382,6 +408,7 @@ class BlockchainEventListenerService { tokenId, keyword, state, + stateIndex, ), this._handleStateFinalizedEvent( TRIPLE_STORE_REPOSITORIES.PRIVATE_CURRENT, @@ -392,6 +419,7 @@ class BlockchainEventListenerService { tokenId, keyword, state, + stateIndex, ), ]); } @@ -406,6 +434,7 @@ class BlockchainEventListenerService { tokenId, keyword, assertionId, + stateIndex, ) { const assertionLinks = await this.tripleStoreService.getAssetAssertionLinks( currentRepository, @@ -465,6 +494,34 @@ class BlockchainEventListenerService { keyword, ), ); + + if ( + currentRepository === TRIPLE_STORE_REPOSITORIES.PUBLIC_CURRENT && + cachedData.agreementId && + cachedData.agreementData + ) { + await this.repositoryModuleManager.updateServiceAgreementRecord( + blockchain, + contract, + tokenId, + cachedData.agreementId, + cachedData.agreementData.startTime, + cachedData.agreementData.epochsNumber, + cachedData.agreementData.epochLength, + cachedData.agreementData.scoreFunctionId, + cachedData.agreementData.proofWindowOffsetPerc, + CONTENT_ASSET_HASH_FUNCTION_ID, + keyword, + assertionId, + stateIndex, + ); + } + } else if (currentRepository === TRIPLE_STORE_REPOSITORIES.PUBLIC_CURRENT) { + await this.repositoryModuleManager.removeServiceAgreementRecord( + blockchain, + contract, + tokenId, + ); } if (cachedData?.private?.assertion && cachedData?.private?.assertionId) { diff --git a/src/service/file-service.js b/src/service/file-service.js index f60481c0a..b428bf18f 100644 --- a/src/service/file-service.js +++ b/src/service/file-service.js @@ -21,8 +21,10 @@ class FileService { * @param data * @returns {Promise} */ - async writeContentsToFile(directory, filename, data) { - this.logger.debug(`Saving file with name: ${filename} in directory: ${directory}`); + async writeContentsToFile(directory, filename, data, log = true) { + if (log) { + this.logger.debug(`Saving file with name: ${filename} in directory: ${directory}`); + } await mkdir(directory, { recursive: true }); const fullpath = path.join(directory, filename); await writeFile(fullpath, data); From eb5480386eadb9971c2f1b98bc418be7063a2416 Mon Sep 17 00:00:00 2001 From: zeroxbt <89495162+zeroxbt@users.noreply.github.com> Date: Wed, 10 May 2023 11:04:47 +0200 Subject: [PATCH 04/10] add better error handling to service agreement migration (#2496) * add better error handling to service agreement migration * lower concurrency to 3 --- .../service-agreements-metadata-migration.js | 56 ++++++++++++++++--- 1 file changed, 48 insertions(+), 8 deletions(-) diff --git a/src/migration/service-agreements-metadata-migration.js b/src/migration/service-agreements-metadata-migration.js index 33e76f4a3..de7c80d9e 100644 --- a/src/migration/service-agreements-metadata-migration.js +++ b/src/migration/service-agreements-metadata-migration.js @@ -1,5 +1,6 @@ /* eslint-disable no-await-in-loop */ import path from 'path'; +import { setTimeout } from 'timers/promises'; import BaseMigration from './base-migration.js'; import { CONTENT_ASSET_HASH_FUNCTION_ID, @@ -55,7 +56,7 @@ class ServiceAgreementsMetadataMigration extends BaseMigration { query, ); const identities = {}; - const concurrency = 10; + const concurrency = 3; let promises = []; let assetsToProcess = assetsMetadata.length; for (const { ual } of assetsMetadata) { @@ -88,10 +89,32 @@ class ServiceAgreementsMetadataMigration extends BaseMigration { } async processAsset(ual, blockchain, contract, tokenId, identityId) { + const maxAttempts = 10; + const sleepTimeSeconds = 2; + // get assertion ids - const assertionIds = await this.blockchainModuleManager - .getAssertionIds(blockchain, contract, tokenId) - .catch(() => {}); + let attempt = 0; + let assertionIds; + while (!assertionIds) { + attempt += 1; + if (attempt >= maxAttempts) + throw Error( + `Error while trying to get assertion ids for asset with ual: ${ual}. Max attempts reached`, + ); + try { + assertionIds = await this.blockchainModuleManager.getAssertionIds( + blockchain, + contract, + tokenId, + ); + } catch (error) { + this.logger.warn( + `Error while trying to get assertion ids for asset with ual: ${ual}. Retrying in ${sleepTimeSeconds} seconds. Attempt number: ${attempt}.`, + ); + } + + await setTimeout(sleepTimeSeconds * 1000); + } if (!assertionIds?.length) { this.logger.warn(`Unable to find assertion ids for asset with ual: ${ual}`); @@ -117,10 +140,27 @@ class ServiceAgreementsMetadataMigration extends BaseMigration { ); // get agreement data - const agreementData = await this.blockchainModuleManager.getAgreementData( - blockchain, - agreementId, - ); + attempt = 0; + let agreementData; + while (!assertionIds) { + attempt += 1; + if (attempt >= maxAttempts) + throw Error( + `Error while trying to get agreement data for asset with ual: ${ual}. Max attempts reached`, + ); + try { + agreementData = await this.blockchainModuleManager.getAgreementData( + blockchain, + agreementId, + ); + } catch (error) { + this.logger.warn( + `Error while trying to get agreement data for asset with ual: ${ual}. Retrying in ${sleepTimeSeconds} seconds. Attempt number: ${attempt}.`, + ); + } + + await setTimeout(sleepTimeSeconds * 1000); + } // calculate current epoch const now = await this.blockchainModuleManager.getBlockchainTimestamp(blockchain); From 9a28728ed0c6c3707aa67f071c6f44ad8a996a00 Mon Sep 17 00:00:00 2001 From: zeroxbt <89495162+zeroxbt@users.noreply.github.com> Date: Wed, 10 May 2023 11:09:54 +0200 Subject: [PATCH 05/10] bump version (#2498) --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 77e0d19c2..68d3e5143 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "origintrail_node", - "version": "6.0.8", + "version": "6.0.8+hotfix.1", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "origintrail_node", - "version": "6.0.8", + "version": "6.0.8+hotfix.1", "license": "ISC", "dependencies": { "@comunica/query-sparql": "^2.4.3", diff --git a/package.json b/package.json index 808d24821..d7c62fd30 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "origintrail_node", - "version": "6.0.8", + "version": "6.0.8+hotfix.1", "description": "OTNode V6", "main": "index.js", "type": "module", From 216dc9b6f205c38869ca1e03fb9bb4aee9a00176 Mon Sep 17 00:00:00 2001 From: zeroxbt <89495162+zeroxbt@users.noreply.github.com> Date: Wed, 10 May 2023 11:51:56 +0200 Subject: [PATCH 06/10] fix service agreement migration (#2500) * fix service agreement migration * bump version to 6.0.8 --- package-lock.json | 4 ++-- package.json | 2 +- src/migration/service-agreements-metadata-migration.js | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/package-lock.json b/package-lock.json index 68d3e5143..77e0d19c2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "origintrail_node", - "version": "6.0.8+hotfix.1", + "version": "6.0.8", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "origintrail_node", - "version": "6.0.8+hotfix.1", + "version": "6.0.8", "license": "ISC", "dependencies": { "@comunica/query-sparql": "^2.4.3", diff --git a/package.json b/package.json index d7c62fd30..808d24821 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "origintrail_node", - "version": "6.0.8+hotfix.1", + "version": "6.0.8", "description": "OTNode V6", "main": "index.js", "type": "module", diff --git a/src/migration/service-agreements-metadata-migration.js b/src/migration/service-agreements-metadata-migration.js index de7c80d9e..9cf8a7ba0 100644 --- a/src/migration/service-agreements-metadata-migration.js +++ b/src/migration/service-agreements-metadata-migration.js @@ -142,7 +142,7 @@ class ServiceAgreementsMetadataMigration extends BaseMigration { // get agreement data attempt = 0; let agreementData; - while (!assertionIds) { + while (!agreementData) { attempt += 1; if (attempt >= maxAttempts) throw Error( From becd70020e08fac951cc893749ed9b5c43ccc223 Mon Sep 17 00:00:00 2001 From: zeroxbt <89495162+zeroxbt@users.noreply.github.com> Date: Wed, 10 May 2023 12:16:22 +0200 Subject: [PATCH 07/10] reduce sleep time in service agreement migration (#2503) --- package-lock.json | 4 ++-- package.json | 2 +- src/migration/service-agreements-metadata-migration.js | 10 ++++++---- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/package-lock.json b/package-lock.json index 77e0d19c2..68d3e5143 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "origintrail_node", - "version": "6.0.8", + "version": "6.0.8+hotfix.1", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "origintrail_node", - "version": "6.0.8", + "version": "6.0.8+hotfix.1", "license": "ISC", "dependencies": { "@comunica/query-sparql": "^2.4.3", diff --git a/package.json b/package.json index 808d24821..d7c62fd30 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "origintrail_node", - "version": "6.0.8", + "version": "6.0.8+hotfix.1", "description": "OTNode V6", "main": "index.js", "type": "module", diff --git a/src/migration/service-agreements-metadata-migration.js b/src/migration/service-agreements-metadata-migration.js index 9cf8a7ba0..a9b81ca46 100644 --- a/src/migration/service-agreements-metadata-migration.js +++ b/src/migration/service-agreements-metadata-migration.js @@ -101,6 +101,9 @@ class ServiceAgreementsMetadataMigration extends BaseMigration { throw Error( `Error while trying to get assertion ids for asset with ual: ${ual}. Max attempts reached`, ); + if (attempt > 1) { + await setTimeout(sleepTimeSeconds * 1000); + } try { assertionIds = await this.blockchainModuleManager.getAssertionIds( blockchain, @@ -112,8 +115,6 @@ class ServiceAgreementsMetadataMigration extends BaseMigration { `Error while trying to get assertion ids for asset with ual: ${ual}. Retrying in ${sleepTimeSeconds} seconds. Attempt number: ${attempt}.`, ); } - - await setTimeout(sleepTimeSeconds * 1000); } if (!assertionIds?.length) { @@ -148,6 +149,9 @@ class ServiceAgreementsMetadataMigration extends BaseMigration { throw Error( `Error while trying to get agreement data for asset with ual: ${ual}. Max attempts reached`, ); + if (attempt > 1) { + await setTimeout(sleepTimeSeconds * 1000); + } try { agreementData = await this.blockchainModuleManager.getAgreementData( blockchain, @@ -158,8 +162,6 @@ class ServiceAgreementsMetadataMigration extends BaseMigration { `Error while trying to get agreement data for asset with ual: ${ual}. Retrying in ${sleepTimeSeconds} seconds. Attempt number: ${attempt}.`, ); } - - await setTimeout(sleepTimeSeconds * 1000); } // calculate current epoch From 52cb8c855a10fd82a56d29e97363a19409fb323a Mon Sep 17 00:00:00 2001 From: zeroxbt <89495162+zeroxbt@users.noreply.github.com> Date: Thu, 11 May 2023 09:48:17 +0200 Subject: [PATCH 08/10] fix mysql disconnects because of inactivity (#2509) * fix mysql disconnects because of inactivity. * bump mysql2 to fix "got packets out of order." err * update dependencies.md --- dependencies.md | 4 +- package-lock.json | 70 +++++++++---------- package.json | 2 +- .../sequelize/sequelize-repository.js | 2 + 4 files changed, 37 insertions(+), 41 deletions(-) diff --git a/dependencies.md b/dependencies.md index 837c5b392..53c7668d8 100644 --- a/dependencies.md +++ b/dependencies.md @@ -151,7 +151,7 @@ ##### [dkg-evm-module](https://www.npmjs.com/package/dkg-evm-module) -- **version**: ^4.0.4 +- **version**: ^4.0.5 - **description**: used to import latest ot-node smart contracts abis ##### [dotenv](https://www.npmjs.com/package/dotenv) @@ -266,7 +266,7 @@ ##### [mysql2](https://www.npmjs.com/package/mysql2) -- **version**: ^2.3.3 +- **version**: ^3.3.0 - **description**: ##### [peer-id](https://www.npmjs.com/package/peer-id) diff --git a/package-lock.json b/package-lock.json index 68d3e5143..ffd1e0cfd 100644 --- a/package-lock.json +++ b/package-lock.json @@ -48,7 +48,7 @@ "libp2p-tcp": "^0.17.2", "minimist": "^1.2.7", "ms": "^2.1.3", - "mysql2": "^2.3.3", + "mysql2": "^3.3.0", "peer-id": "^0.15.3", "pino": "^8.4.2", "pino-pretty": "^9.1.0", @@ -14105,16 +14105,16 @@ } }, "node_modules/mysql2": { - "version": "2.3.3", - "resolved": "https://registry.npmjs.org/mysql2/-/mysql2-2.3.3.tgz", - "integrity": "sha512-wxJUev6LgMSgACDkb/InIFxDprRa6T95+VEoR+xPvtngtccNH2dGjEB/fVZ8yg1gWv1510c9CvXuJHi5zUm0ZA==", + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/mysql2/-/mysql2-3.3.0.tgz", + "integrity": "sha512-/+LGlWgXxppcomT1NqkUnaDchcS9tebsXsj5eZQhnAB/onlSLgqMA5W9ZRHcZPKyqg3XROJDgomB4eCkn6Ca2g==", "dependencies": { - "denque": "^2.0.1", + "denque": "^2.1.0", "generate-function": "^2.3.1", "iconv-lite": "^0.6.3", - "long": "^4.0.0", - "lru-cache": "^6.0.0", - "named-placeholders": "^1.1.2", + "long": "^5.2.1", + "lru-cache": "^8.0.0", + "named-placeholders": "^1.1.3", "seq-queue": "^0.0.5", "sqlstring": "^2.3.2" }, @@ -14133,22 +14133,19 @@ "node": ">=0.10.0" } }, + "node_modules/mysql2/node_modules/long": { + "version": "5.2.3", + "resolved": "https://registry.npmjs.org/long/-/long-5.2.3.tgz", + "integrity": "sha512-lcHwpNoggQTObv5apGNCTdJrO69eHOZMi4BNC+rTLER8iHAqGrUVeLh/irVIM7zTw2bOXA8T6uNPeujwOLg/2Q==" + }, "node_modules/mysql2/node_modules/lru-cache": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", - "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", - "dependencies": { - "yallist": "^4.0.0" - }, + "version": "8.0.5", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-8.0.5.tgz", + "integrity": "sha512-MhWWlVnuab1RG5/zMRRcVGXZLCXrZTgfwMikgzCegsPnG62yDQo5JnqKkrK4jO5iKqDAZGItAqN5CtKBCBWRUA==", "engines": { - "node": ">=10" + "node": ">=16.14" } }, - "node_modules/mysql2/node_modules/yallist": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", - "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" - }, "node_modules/mz": { "version": "2.7.0", "resolved": "https://registry.npmjs.org/mz/-/mz-2.7.0.tgz", @@ -30591,16 +30588,16 @@ "integrity": "sha512-4OvNRr1DJpy2QuDUV74m+BWZ//n4gG4bmd21MzDSPqHEidIDWqwyOjcadU1LBMO3vXYGurVKjfBrxrSQIHFu9A==" }, "mysql2": { - "version": "2.3.3", - "resolved": "https://registry.npmjs.org/mysql2/-/mysql2-2.3.3.tgz", - "integrity": "sha512-wxJUev6LgMSgACDkb/InIFxDprRa6T95+VEoR+xPvtngtccNH2dGjEB/fVZ8yg1gWv1510c9CvXuJHi5zUm0ZA==", + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/mysql2/-/mysql2-3.3.0.tgz", + "integrity": "sha512-/+LGlWgXxppcomT1NqkUnaDchcS9tebsXsj5eZQhnAB/onlSLgqMA5W9ZRHcZPKyqg3XROJDgomB4eCkn6Ca2g==", "requires": { - "denque": "^2.0.1", + "denque": "^2.1.0", "generate-function": "^2.3.1", "iconv-lite": "^0.6.3", - "long": "^4.0.0", - "lru-cache": "^6.0.0", - "named-placeholders": "^1.1.2", + "long": "^5.2.1", + "lru-cache": "^8.0.0", + "named-placeholders": "^1.1.3", "seq-queue": "^0.0.5", "sqlstring": "^2.3.2" }, @@ -30613,18 +30610,15 @@ "safer-buffer": ">= 2.1.2 < 3.0.0" } }, - "lru-cache": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", - "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", - "requires": { - "yallist": "^4.0.0" - } + "long": { + "version": "5.2.3", + "resolved": "https://registry.npmjs.org/long/-/long-5.2.3.tgz", + "integrity": "sha512-lcHwpNoggQTObv5apGNCTdJrO69eHOZMi4BNC+rTLER8iHAqGrUVeLh/irVIM7zTw2bOXA8T6uNPeujwOLg/2Q==" }, - "yallist": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", - "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" + "lru-cache": { + "version": "8.0.5", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-8.0.5.tgz", + "integrity": "sha512-MhWWlVnuab1RG5/zMRRcVGXZLCXrZTgfwMikgzCegsPnG62yDQo5JnqKkrK4jO5iKqDAZGItAqN5CtKBCBWRUA==" } } }, diff --git a/package.json b/package.json index d7c62fd30..2842da0b1 100644 --- a/package.json +++ b/package.json @@ -97,7 +97,7 @@ "libp2p-tcp": "^0.17.2", "minimist": "^1.2.7", "ms": "^2.1.3", - "mysql2": "^2.3.3", + "mysql2": "^3.3.0", "peer-id": "^0.15.3", "pino": "^8.4.2", "pino-pretty": "^9.1.0", diff --git a/src/modules/repository/implementation/sequelize/sequelize-repository.js b/src/modules/repository/implementation/sequelize/sequelize-repository.js index a61cae5ad..04a2f800b 100644 --- a/src/modules/repository/implementation/sequelize/sequelize-repository.js +++ b/src/modules/repository/implementation/sequelize/sequelize-repository.js @@ -58,6 +58,7 @@ class SequelizeRepository { await connection .promise() .query(`CREATE DATABASE IF NOT EXISTS \`${this.config.database}\`;`); + connection.destroy(); } async dropDatabase() { @@ -68,6 +69,7 @@ class SequelizeRepository { password: process.env.SEQUELIZE_REPOSITORY_PASSWORD, }); await connection.promise().query(`DROP DATABASE IF EXISTS \`${this.config.database}\`;`); + connection.destroy(); } async runMigrations() { From 5ce4e2a3bbb7e54a4d55e139aa0a4c6d28a186a0 Mon Sep 17 00:00:00 2001 From: zeroxbt <89495162+zeroxbt@users.noreply.github.com> Date: Thu, 11 May 2023 09:48:47 +0200 Subject: [PATCH 09/10] Fix switch rpc on error (#2508) --- .../implementation/ot-parachain/ot-parachain-service.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/modules/blockchain/implementation/ot-parachain/ot-parachain-service.js b/src/modules/blockchain/implementation/ot-parachain/ot-parachain-service.js index 4066ecc0d..77f5854a5 100644 --- a/src/modules/blockchain/implementation/ot-parachain/ot-parachain-service.js +++ b/src/modules/blockchain/implementation/ot-parachain/ot-parachain-service.js @@ -86,13 +86,13 @@ class OtParachainService extends Web3Service { provider = new HttpProvider(this.config.rpcEndpoints[this.rpcNumber]); } // eslint-disable-next-line no-await-in-loop - this.parachainProvider = await new ApiPromise({ provider }).isReady; + this.parachainProvider = await new ApiPromise({ provider }).isReadyOrError; isRpcConnected = true; } catch (e) { this.logger.warn( `Unable to create parachain provider for endpoint : ${ this.config.rpcEndpoints[this.rpcNumber] - }.`, + }. Error: ${e.message}`, ); tries += 1; this.rpcNumber = (this.rpcNumber + 1) % this.config.rpcEndpoints.length; From 3cb5f9362175ce9634abc51d2e4a279df82b040b Mon Sep 17 00:00:00 2001 From: zeroxbt <89495162+zeroxbt@users.noreply.github.com> Date: Thu, 11 May 2023 09:49:11 +0200 Subject: [PATCH 10/10] handle errors when getting commits in service agreement migration (#2507) * handle errors when getting commits in SA migration * bump version to 6.0.8 --- package-lock.json | 4 +-- package.json | 2 +- .../service-agreements-metadata-migration.js | 31 +++++++++++++++---- 3 files changed, 28 insertions(+), 9 deletions(-) diff --git a/package-lock.json b/package-lock.json index ffd1e0cfd..7cb3b52d7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "origintrail_node", - "version": "6.0.8+hotfix.1", + "version": "6.0.8", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "origintrail_node", - "version": "6.0.8+hotfix.1", + "version": "6.0.8", "license": "ISC", "dependencies": { "@comunica/query-sparql": "^2.4.3", diff --git a/package.json b/package.json index 2842da0b1..8f2ef8015 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "origintrail_node", - "version": "6.0.8+hotfix.1", + "version": "6.0.8", "description": "OTNode V6", "main": "index.js", "type": "module", diff --git a/src/migration/service-agreements-metadata-migration.js b/src/migration/service-agreements-metadata-migration.js index a9b81ca46..57a71d077 100644 --- a/src/migration/service-agreements-metadata-migration.js +++ b/src/migration/service-agreements-metadata-migration.js @@ -172,12 +172,31 @@ class ServiceAgreementsMetadataMigration extends BaseMigration { if (epoch >= Number(agreementData.epochsNumber)) return; // get top commits - const commits = await this.blockchainModuleManager.getTopCommitSubmissions( - blockchain, - agreementId, - epoch, - stateIndex, - ); + attempt = 0; + let commits; + while (!commits) { + attempt += 1; + if (attempt >= maxAttempts) + throw Error( + `Error while trying to get top commit submissions for asset with ual: ${ual}. Max attempts reached`, + ); + if (attempt > 1) { + await setTimeout(sleepTimeSeconds * 1000); + } + try { + commits = await this.blockchainModuleManager.getTopCommitSubmissions( + blockchain, + agreementId, + epoch, + stateIndex, + ); + } catch (error) { + this.logger.warn( + `Error while trying to get top commit submissions for asset with ual: ${ual}. Retrying in ${sleepTimeSeconds} seconds. Attempt number: ${attempt}.`, + ); + } + } + let lastCommitEpoch = null; let lastProofEpoch = null;