From 9adb2d45c2966876b1cc257bc59be09a9f59c461 Mon Sep 17 00:00:00 2001 From: zeroxbt <89495162+zeroxbt@users.noreply.github.com> Date: Tue, 23 May 2023 13:54:01 +0200 Subject: [PATCH 01/33] reduce blockchain calls in epoch check command (#2561) --- .../protocols/common/epoch-check-command.js | 31 ++++++++++++++----- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/src/commands/protocols/common/epoch-check-command.js b/src/commands/protocols/common/epoch-check-command.js index 1ab33426f..f9e3730c9 100644 --- a/src/commands/protocols/common/epoch-check-command.js +++ b/src/commands/protocols/common/epoch-check-command.js @@ -40,18 +40,27 @@ class EpochCheckCommand extends Command { const transactionQueueLength = this.blockchainModuleManager.getTransactionQueueLength(blockchain); if (transactionQueueLength >= totalTransactions) return; + totalTransactions -= transactionQueueLength; + const [r0, r2] = await Promise.all([ + this.blockchainModuleManager.getR0(blockchain), + this.blockchainModuleManager.getR2(blockchain), + ]); + await Promise.all([ this.scheduleSubmitCommitCommands( blockchain, Math.floor(totalTransactions / 2), commitWindowDurationPerc, + r0, + r2, ), this.scheduleCalculateProofsCommands( blockchain, Math.ceil(totalTransactions / 2), proofWindowDurationPerc, + r0, ), ]); }), @@ -59,7 +68,13 @@ class EpochCheckCommand extends Command { return Command.repeat(); } - async scheduleSubmitCommitCommands(blockchain, maxTransactions, commitWindowDurationPerc) { + async scheduleSubmitCommitCommands( + blockchain, + maxTransactions, + commitWindowDurationPerc, + r0, + r2, + ) { const timestamp = await this.blockchainModuleManager.getBlockchainTimestamp(blockchain); const eligibleAgreementForSubmitCommit = await this.repositoryModuleManager.getEligibleAgreementsForSubmitCommit( @@ -68,9 +83,6 @@ class EpochCheckCommand extends Command { commitWindowDurationPerc, ); - const r0 = await this.blockchainModuleManager.getR0(blockchain); - const r2 = await this.blockchainModuleManager.getR2(blockchain); - const scheduleSubmitCommitCommands = []; const updateServiceAgreementsLastCommitEpoch = []; for (const serviceAgreement of eligibleAgreementForSubmitCommit) { @@ -120,7 +132,12 @@ class EpochCheckCommand extends Command { ]); } - async scheduleCalculateProofsCommands(blockchain, maxTransactions, proofWindowDurationPerc) { + async scheduleCalculateProofsCommands( + blockchain, + maxTransactions, + proofWindowDurationPerc, + r0, + ) { const timestamp = await this.blockchainModuleManager.getBlockchainTimestamp(blockchain); const eligibleAgreementsForSubmitProofs = await this.repositoryModuleManager.getEligibleAgreementsForSubmitProof( @@ -138,6 +155,7 @@ class EpochCheckCommand extends Command { serviceAgreement.agreementId, serviceAgreement.currentEpoch, serviceAgreement.stateIndex, + r0, ); if (eligibleForReward) { this.logger.trace( @@ -196,8 +214,7 @@ class EpochCheckCommand extends Command { return scores.findIndex((node) => node.peerId === peerId); } - async isEligibleForRewards(blockchain, agreementId, epoch, stateIndex) { - const r0 = await this.blockchainModuleManager.getR0(blockchain); + async isEligibleForRewards(blockchain, agreementId, epoch, stateIndex, r0) { const identityId = await this.blockchainModuleManager.getIdentityId(blockchain); const commits = await this.blockchainModuleManager.getTopCommitSubmissions( blockchain, From d71dbe873e43e5c7f7db3ef7c57cc7299e58dfff Mon Sep 17 00:00:00 2001 From: zeroxbt <89495162+zeroxbt@users.noreply.github.com> Date: Tue, 23 May 2023 14:06:53 +0200 Subject: [PATCH 02/33] fix filtering of pending commands during replay (#2539) --- src/commands/command-executor.js | 106 +++++++++++++++---------------- 1 file changed, 51 insertions(+), 55 deletions(-) diff --git a/src/commands/command-executor.js b/src/commands/command-executor.js index e44734f71..f1f919e8e 100644 --- a/src/commands/command-executor.js +++ b/src/commands/command-executor.js @@ -14,14 +14,12 @@ import { */ class CommandExecutor { constructor(ctx) { - this.ctx = ctx; this.logger = ctx.logger; this.commandResolver = ctx.commandResolver; - this.config = ctx.config; this.started = false; this.repositoryModuleManager = ctx.repositoryModuleManager; - this.verboseLoggingEnabled = this.config.commandExecutorVerboseLoggingEnabled; + this.verboseLoggingEnabled = ctx.config.commandExecutorVerboseLoggingEnabled; this.queue = async.queue((command, callback = () => {}) => { this._execute(command) @@ -90,8 +88,8 @@ class CommandExecutor { }); try { const result = await handler.expired(command); - if (result && result.commands) { - result.commands.forEach((c) => this.add(c, c.delay, true)); + if (result?.commands) { + await Promise.all(result.commands.map((c) => this.add(c, c.delay, true))); } } catch (e) { this.logger.warn( @@ -135,9 +133,7 @@ class CommandExecutor { command.data = handler.pack(command.data); - const period = command.period - ? command.period - : DEFAULT_COMMAND_REPEAT_INTERVAL_IN_MILLS; + const period = command.period ?? DEFAULT_COMMAND_REPEAT_INTERVAL_IN_MILLS; await this.add(command, period, false); return Command.repeat(); } @@ -187,8 +183,8 @@ class CommandExecutor { try { const result = await this._handleError(command, handler, e); - if (result && result.commands) { - result.commands.forEach((c) => this.add(c, c.delay, true)); + if (result?.commands) { + await Promise.all(result.commands.map((c) => this.add(c, c.delay, true))); } } catch (error) { this.logger.warn( @@ -237,14 +233,13 @@ class CommandExecutor { * @param delay * @param insert */ - async add(addCommand, addDelay = 0, insert = true) { + async add(addCommand, addDelay, insert = true) { let command = addCommand; - let delay = addDelay; - const now = Date.now(); + let delay = addDelay ?? 0; - if (delay != null && delay > MAX_COMMAND_DELAY_IN_MILLS) { + if (delay > MAX_COMMAND_DELAY_IN_MILLS) { if (command.readyAt == null) { - command.readyAt = now; + command.readyAt = Date.now(); } command.readyAt += delay; delay = MAX_COMMAND_DELAY_IN_MILLS; @@ -280,8 +275,8 @@ class CommandExecutor { status: COMMAND_STATUS.PENDING, retries: command.retries - 1, }); - const period = command.period ? command.period : 0; - const delay = command.delay ? command.delay : 0; + const period = command.period ?? 0; + const delay = command.delay ?? 0; await this.add(command, period + delay, false); return Command.retry(); } @@ -302,8 +297,8 @@ class CommandExecutor { await this._update(command, { retries: command.retries - 1, }); - const period = command.period ? command.period : 0; - const delay = command.delay ? command.delay : 0; + const period = command.period ?? 0; + const delay = command.delay ?? 0; await this.add(command, period + delay, false); } else { try { @@ -353,7 +348,7 @@ class CommandExecutor { opts.transaction = transaction; } const model = await this.repositoryModuleManager.createCommand(command, opts); - command.id = model.dataValues.id; + command.id = model.id; return command; } @@ -394,49 +389,50 @@ class CommandExecutor { * @returns {Promise} */ async replay() { - // Wait for 1 minute for node to establish connections - // await new Promise((resolve) => setTimeout(resolve, 1 * 60 * 1000)); - this.logger.info('Replay pending/started commands from the database...'); - const pendingCommands = ( - await this.repositoryModuleManager.getCommandsWithStatus( - [COMMAND_STATUS.PENDING, COMMAND_STATUS.STARTED, COMMAND_STATUS.REPEATING], - ['cleanerCommand', 'autoupdaterCommand'], - ) - ).filter((command) => !PERMANENT_COMMANDS.includes(command.name)); - - // TODO consider JOIN instead - const commands = pendingCommands.filter(async (pc) => { - if (!pc.parentId) { - return true; + const pendingCommands = await this.repositoryModuleManager.getCommandsWithStatus( + [COMMAND_STATUS.PENDING, COMMAND_STATUS.STARTED, COMMAND_STATUS.REPEATING], + PERMANENT_COMMANDS, + ); + + const commands = []; + for (const command of pendingCommands) { + if (!command?.parentId) { + continue; } - const parent = await this.repositoryModuleManager.getCommandWithId(pc.parentId); - return !parent || parent.status === 'COMPLETED'; - }); + + // eslint-disable-next-line no-await-in-loop + const parent = await this.repositoryModuleManager.getCommandWithId(command.parentId); + if (parent && parent.status !== 'COMPLETED') { + continue; + } + commands.push(command); + } const adds = []; for (const commandModel of commands) { - const command = { - id: commandModel.id, - name: commandModel.name, - data: commandModel.data, - readyAt: commandModel.readyAt, - delay: commandModel.delay, - startedAt: commandModel.startedAt, - deadlineAt: commandModel.deadlineAt, - period: commandModel.period, - status: commandModel.status, - message: commandModel.message, - parentId: commandModel.parentId, - transactional: commandModel.transactional, - retries: commandModel.retries, - sequence: commandModel.sequence, - }; - const queued = this.queue.workersList().find((e) => e.data.id === command.id); + const queued = this.queue.workersList().find((e) => e.data.id === commandModel.id); if (!queued) { - adds.push(this.add(command, 0, false, true)); + const command = { + id: commandModel.id, + name: commandModel.name, + data: commandModel.data, + readyAt: commandModel.readyAt, + delay: commandModel.delay, + startedAt: commandModel.startedAt, + deadlineAt: commandModel.deadlineAt, + period: commandModel.period, + status: commandModel.status, + message: commandModel.message, + parentId: commandModel.parentId, + transactional: commandModel.transactional, + retries: commandModel.retries, + sequence: commandModel.sequence, + }; + adds.push(this.add(command, 0, false)); } } + await Promise.all(adds); } } From 25e4ad2cd448e8988a99596b67a062e462c852a5 Mon Sep 17 00:00:00 2001 From: djordjekovac Date: Fri, 26 May 2023 15:47:08 +0200 Subject: [PATCH 03/33] Fixed bug when command is not repeated during recovery (#2565) * Fixed bug when command is not repeated during recovery * Update command-executor.js --- src/commands/command-executor.js | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/commands/command-executor.js b/src/commands/command-executor.js index f1f919e8e..e62e042ac 100644 --- a/src/commands/command-executor.js +++ b/src/commands/command-executor.js @@ -183,8 +183,18 @@ class CommandExecutor { try { const result = await this._handleError(command, handler, e); - if (result?.commands) { - await Promise.all(result.commands.map((c) => this.add(c, c.delay, true))); + if (result && result.repeat) { + await this._update(command, { + status: COMMAND_STATUS.REPEATING, + }); + + command.data = handler.pack(command.data); + + const period = command.period + ? command.period + : DEFAULT_COMMAND_REPEAT_INTERVAL_IN_MILLS; + await this.add(command, period, false); + return Command.repeat(); } } catch (error) { this.logger.warn( From 6cb076bf32095c8260194ecb802aa34a0cec8a0e Mon Sep 17 00:00:00 2001 From: zeroxbt Date: Sat, 27 May 2023 19:22:10 +0200 Subject: [PATCH 04/33] only fetch necessary events --- src/constants/constants.js | 32 ++++------------ .../blockchain/blockchain-module-manager.js | 2 + .../blockchain/implementation/web3-service.js | 21 ++++++++-- .../blockchain-event-listener-service.js | 38 ++++++++++++++----- 4 files changed, 56 insertions(+), 37 deletions(-) diff --git a/src/constants/constants.js b/src/constants/constants.js index 611ded831..0a142062e 100644 --- a/src/constants/constants.js +++ b/src/constants/constants.js @@ -27,7 +27,7 @@ export const TRIPLE_STORE_CONNECT_MAX_RETRIES = 10; export const DEFAULT_BLOCKCHAIN_EVENT_SYNC_PERIOD_IN_MILLS = 15 * 24 * 60 * 60 * 1000; // 15 days -export const MAXIMUM_NUMBERS_OF_BLOCKS_TO_FETCH = 500; +export const MAXIMUM_NUMBERS_OF_BLOCKS_TO_FETCH = 50; export const TRANSACTION_QUEUE_CONCURRENCY = 1; @@ -415,30 +415,12 @@ export const CONTRACTS = { }; export const CONTRACT_EVENTS = { - HUB: { - NEW_CONTRACT: 'NewContract', - CONTRACT_CHANGED: 'ContractChanged', - NEW_ASSET_STORAGE: 'NewAssetStorage', - ASSET_STORAGE_CHANGED: 'AssetStorageChanged', - }, - SHARDING_TABLE: { - NODE_ADDED: 'NodeAdded', - NODE_REMOVED: 'NodeRemoved', - }, - STAKING: { - STAKE_INCREASED: 'StakeIncreased', - STAKE_WITHDRAWAL_STARTED: 'StakeWithdrawalStarted', - }, - PROFILE: { - ASK_UPDATED: 'AskUpdated', - }, - COMMIT_MANAGER_V1: { - STATE_FINALIZED: 'StateFinalized', - }, - SERVICE_AGREEMENT_V1: { - SERVICE_AGREEMENT_V1_EXTENDED: 'ServiceAgreementV1Extended', - SERVICE_AGREEMENT_V1_TERMINATED: 'ServiceAgreementV1Terminated', - }, + HUB: ['NewContract', 'ContractChanged', 'NewAssetStorage', 'AssetStorageChanged'], + SHARDING_TABLE: ['NodeAdded', 'NodeRemoved'], + STAKING: ['StakeIncreased', 'StakeWithdrawalStarted'], + PROFILE: ['AskUpdated'], + COMMIT_MANAGER_V1: ['StateFinalized'], + SERVICE_AGREEMENT_V1: ['ServiceAgreementV1Extended', 'ServiceAgreementV1Terminated'], }; export const NODE_ENVIRONMENTS = { diff --git a/src/modules/blockchain/blockchain-module-manager.js b/src/modules/blockchain/blockchain-module-manager.js index 2079c1327..53c32c0d9 100644 --- a/src/modules/blockchain/blockchain-module-manager.js +++ b/src/modules/blockchain/blockchain-module-manager.js @@ -141,6 +141,7 @@ class BlockchainModuleManager extends BaseModuleManager { async getAllPastEvents( blockchain, contractName, + eventsToFilter, lastCheckedBlock, lastCheckedTimestamp, currentBlock, @@ -148,6 +149,7 @@ class BlockchainModuleManager extends BaseModuleManager { return this.callImplementationFunction(blockchain, 'getAllPastEvents', [ blockchain, contractName, + eventsToFilter, lastCheckedBlock, lastCheckedTimestamp, currentBlock, diff --git a/src/modules/blockchain/implementation/web3-service.js b/src/modules/blockchain/implementation/web3-service.js index 54f122b55..946a56202 100644 --- a/src/modules/blockchain/implementation/web3-service.js +++ b/src/modules/blockchain/implementation/web3-service.js @@ -396,6 +396,7 @@ class Web3Service { async getAllPastEvents( blockchainId, contractName, + eventsToFilter, lastCheckedBlock, lastCheckedTimestamp, currentBlock, @@ -412,14 +413,21 @@ class Web3Service { fromBlock = lastCheckedBlock + 1; } - let events = []; + const topics = []; + for (const filterName in contract.filters) { + if (!eventsToFilter.includes(filterName)) continue; + const filter = contract.filters[filterName]().topics[0]; + topics.push(filter); + } + + const events = []; while (fromBlock <= currentBlock) { const toBlock = Math.min( fromBlock + MAXIMUM_NUMBERS_OF_BLOCKS_TO_FETCH - 1, currentBlock, ); - const newEvents = await contract.queryFilter('*', fromBlock, toBlock); - events = events.concat(newEvents); + const newEvents = await this.processBlockRange(fromBlock, toBlock, contract, topics); + newEvents.forEach((e) => events.push(...e)); fromBlock = toBlock + 1; } @@ -439,6 +447,13 @@ class Web3Service { })); } + async processBlockRange(fromBlock, toBlock, contract, topics) { + const newEvents = await Promise.all( + topics.map((topic) => contract.queryFilter(topic, fromBlock, toBlock)), + ); + return newEvents; + } + isOlderThan(timestamp, olderThanInMills) { if (!timestamp) return true; const timestampThirtyDaysInPast = new Date().getTime() - olderThanInMills; diff --git a/src/service/blockchain-event-listener-service.js b/src/service/blockchain-event-listener-service.js index a6a458139..72e820208 100644 --- a/src/service/blockchain-event-listener-service.js +++ b/src/service/blockchain-event-listener-service.js @@ -11,10 +11,7 @@ import { const MAXIMUM_FETCH_EVENTS_FAILED_COUNT = 5; const fetchEventsFailedCount = {}; -const eventNames = []; -Object.keys(CONTRACT_EVENTS).forEach((contractName) => { - eventNames.push(...Object.values(CONTRACT_EVENTS[contractName])); -}); +const eventNames = Object.values(CONTRACT_EVENTS).flatMap((e) => e); class BlockchainEventListenerService { constructor(ctx) { @@ -52,24 +49,46 @@ class BlockchainEventListenerService { const currentBlock = await this.blockchainModuleManager.getBlockNumber(); const syncContractEventsPromises = [ - this.getContractEvents(blockchainId, CONTRACTS.SHARDING_TABLE_CONTRACT, currentBlock), - this.getContractEvents(blockchainId, CONTRACTS.STAKING_CONTRACT, currentBlock), - this.getContractEvents(blockchainId, CONTRACTS.PROFILE_CONTRACT, currentBlock), + this.getContractEvents( + blockchainId, + CONTRACTS.SHARDING_TABLE_CONTRACT, + currentBlock, + CONTRACT_EVENTS.SHARDING_TABLE, + ), + this.getContractEvents( + blockchainId, + CONTRACTS.STAKING_CONTRACT, + currentBlock, + CONTRACT_EVENTS.STAKING, + ), + this.getContractEvents( + blockchainId, + CONTRACTS.PROFILE_CONTRACT, + currentBlock, + CONTRACT_EVENTS.PROFILE, + ), this.getContractEvents( blockchainId, CONTRACTS.COMMIT_MANAGER_V1_U1_CONTRACT, currentBlock, + CONTRACT_EVENTS.COMMIT_MANAGER_V1, ), this.getContractEvents( blockchainId, CONTRACTS.SERVICE_AGREEMENT_V1_CONTRACT, currentBlock, + CONTRACT_EVENTS.SERVICE_AGREEMENT_V1, ), ]; if (!devEnvironment) { syncContractEventsPromises.push( - this.getContractEvents(blockchainId, CONTRACTS.HUB_CONTRACT, currentBlock), + this.getContractEvents( + blockchainId, + CONTRACTS.HUB_CONTRACT, + currentBlock, + CONTRACT_EVENTS.HUB, + ), ); } const contractEvents = await Promise.all(syncContractEventsPromises); @@ -118,7 +137,7 @@ class BlockchainEventListenerService { }, eventFetchInterval); } - async getContractEvents(blockchainId, contractName, currentBlock) { + async getContractEvents(blockchainId, contractName, currentBlock, eventsToFilter) { const lastCheckedBlockObject = await this.repositoryModuleManager.getLastCheckedBlock( blockchainId, contractName, @@ -127,6 +146,7 @@ class BlockchainEventListenerService { const events = await this.blockchainModuleManager.getAllPastEvents( blockchainId, contractName, + eventsToFilter, lastCheckedBlockObject?.lastCheckedBlock ?? 0, lastCheckedBlockObject?.lastCheckedTimestamp ?? 0, currentBlock, From 9a2cb1014d56d6ad11bbd5e1a2b34f5ecf97ed96 Mon Sep 17 00:00:00 2001 From: zeroxbt Date: Sat, 27 May 2023 19:29:42 +0200 Subject: [PATCH 05/33] Revert "only fetch necessary events" This reverts commit 6cb076bf32095c8260194ecb802aa34a0cec8a0e. --- src/constants/constants.js | 32 ++++++++++++---- .../blockchain/blockchain-module-manager.js | 2 - .../blockchain/implementation/web3-service.js | 21 ++-------- .../blockchain-event-listener-service.js | 38 +++++-------------- 4 files changed, 37 insertions(+), 56 deletions(-) diff --git a/src/constants/constants.js b/src/constants/constants.js index 0a142062e..611ded831 100644 --- a/src/constants/constants.js +++ b/src/constants/constants.js @@ -27,7 +27,7 @@ export const TRIPLE_STORE_CONNECT_MAX_RETRIES = 10; export const DEFAULT_BLOCKCHAIN_EVENT_SYNC_PERIOD_IN_MILLS = 15 * 24 * 60 * 60 * 1000; // 15 days -export const MAXIMUM_NUMBERS_OF_BLOCKS_TO_FETCH = 50; +export const MAXIMUM_NUMBERS_OF_BLOCKS_TO_FETCH = 500; export const TRANSACTION_QUEUE_CONCURRENCY = 1; @@ -415,12 +415,30 @@ export const CONTRACTS = { }; export const CONTRACT_EVENTS = { - HUB: ['NewContract', 'ContractChanged', 'NewAssetStorage', 'AssetStorageChanged'], - SHARDING_TABLE: ['NodeAdded', 'NodeRemoved'], - STAKING: ['StakeIncreased', 'StakeWithdrawalStarted'], - PROFILE: ['AskUpdated'], - COMMIT_MANAGER_V1: ['StateFinalized'], - SERVICE_AGREEMENT_V1: ['ServiceAgreementV1Extended', 'ServiceAgreementV1Terminated'], + HUB: { + NEW_CONTRACT: 'NewContract', + CONTRACT_CHANGED: 'ContractChanged', + NEW_ASSET_STORAGE: 'NewAssetStorage', + ASSET_STORAGE_CHANGED: 'AssetStorageChanged', + }, + SHARDING_TABLE: { + NODE_ADDED: 'NodeAdded', + NODE_REMOVED: 'NodeRemoved', + }, + STAKING: { + STAKE_INCREASED: 'StakeIncreased', + STAKE_WITHDRAWAL_STARTED: 'StakeWithdrawalStarted', + }, + PROFILE: { + ASK_UPDATED: 'AskUpdated', + }, + COMMIT_MANAGER_V1: { + STATE_FINALIZED: 'StateFinalized', + }, + SERVICE_AGREEMENT_V1: { + SERVICE_AGREEMENT_V1_EXTENDED: 'ServiceAgreementV1Extended', + SERVICE_AGREEMENT_V1_TERMINATED: 'ServiceAgreementV1Terminated', + }, }; export const NODE_ENVIRONMENTS = { diff --git a/src/modules/blockchain/blockchain-module-manager.js b/src/modules/blockchain/blockchain-module-manager.js index 53c32c0d9..2079c1327 100644 --- a/src/modules/blockchain/blockchain-module-manager.js +++ b/src/modules/blockchain/blockchain-module-manager.js @@ -141,7 +141,6 @@ class BlockchainModuleManager extends BaseModuleManager { async getAllPastEvents( blockchain, contractName, - eventsToFilter, lastCheckedBlock, lastCheckedTimestamp, currentBlock, @@ -149,7 +148,6 @@ class BlockchainModuleManager extends BaseModuleManager { return this.callImplementationFunction(blockchain, 'getAllPastEvents', [ blockchain, contractName, - eventsToFilter, lastCheckedBlock, lastCheckedTimestamp, currentBlock, diff --git a/src/modules/blockchain/implementation/web3-service.js b/src/modules/blockchain/implementation/web3-service.js index 946a56202..54f122b55 100644 --- a/src/modules/blockchain/implementation/web3-service.js +++ b/src/modules/blockchain/implementation/web3-service.js @@ -396,7 +396,6 @@ class Web3Service { async getAllPastEvents( blockchainId, contractName, - eventsToFilter, lastCheckedBlock, lastCheckedTimestamp, currentBlock, @@ -413,21 +412,14 @@ class Web3Service { fromBlock = lastCheckedBlock + 1; } - const topics = []; - for (const filterName in contract.filters) { - if (!eventsToFilter.includes(filterName)) continue; - const filter = contract.filters[filterName]().topics[0]; - topics.push(filter); - } - - const events = []; + let events = []; while (fromBlock <= currentBlock) { const toBlock = Math.min( fromBlock + MAXIMUM_NUMBERS_OF_BLOCKS_TO_FETCH - 1, currentBlock, ); - const newEvents = await this.processBlockRange(fromBlock, toBlock, contract, topics); - newEvents.forEach((e) => events.push(...e)); + const newEvents = await contract.queryFilter('*', fromBlock, toBlock); + events = events.concat(newEvents); fromBlock = toBlock + 1; } @@ -447,13 +439,6 @@ class Web3Service { })); } - async processBlockRange(fromBlock, toBlock, contract, topics) { - const newEvents = await Promise.all( - topics.map((topic) => contract.queryFilter(topic, fromBlock, toBlock)), - ); - return newEvents; - } - isOlderThan(timestamp, olderThanInMills) { if (!timestamp) return true; const timestampThirtyDaysInPast = new Date().getTime() - olderThanInMills; diff --git a/src/service/blockchain-event-listener-service.js b/src/service/blockchain-event-listener-service.js index 72e820208..a6a458139 100644 --- a/src/service/blockchain-event-listener-service.js +++ b/src/service/blockchain-event-listener-service.js @@ -11,7 +11,10 @@ import { const MAXIMUM_FETCH_EVENTS_FAILED_COUNT = 5; const fetchEventsFailedCount = {}; -const eventNames = Object.values(CONTRACT_EVENTS).flatMap((e) => e); +const eventNames = []; +Object.keys(CONTRACT_EVENTS).forEach((contractName) => { + eventNames.push(...Object.values(CONTRACT_EVENTS[contractName])); +}); class BlockchainEventListenerService { constructor(ctx) { @@ -49,46 +52,24 @@ class BlockchainEventListenerService { const currentBlock = await this.blockchainModuleManager.getBlockNumber(); const syncContractEventsPromises = [ - this.getContractEvents( - blockchainId, - CONTRACTS.SHARDING_TABLE_CONTRACT, - currentBlock, - CONTRACT_EVENTS.SHARDING_TABLE, - ), - this.getContractEvents( - blockchainId, - CONTRACTS.STAKING_CONTRACT, - currentBlock, - CONTRACT_EVENTS.STAKING, - ), - this.getContractEvents( - blockchainId, - CONTRACTS.PROFILE_CONTRACT, - currentBlock, - CONTRACT_EVENTS.PROFILE, - ), + this.getContractEvents(blockchainId, CONTRACTS.SHARDING_TABLE_CONTRACT, currentBlock), + this.getContractEvents(blockchainId, CONTRACTS.STAKING_CONTRACT, currentBlock), + this.getContractEvents(blockchainId, CONTRACTS.PROFILE_CONTRACT, currentBlock), this.getContractEvents( blockchainId, CONTRACTS.COMMIT_MANAGER_V1_U1_CONTRACT, currentBlock, - CONTRACT_EVENTS.COMMIT_MANAGER_V1, ), this.getContractEvents( blockchainId, CONTRACTS.SERVICE_AGREEMENT_V1_CONTRACT, currentBlock, - CONTRACT_EVENTS.SERVICE_AGREEMENT_V1, ), ]; if (!devEnvironment) { syncContractEventsPromises.push( - this.getContractEvents( - blockchainId, - CONTRACTS.HUB_CONTRACT, - currentBlock, - CONTRACT_EVENTS.HUB, - ), + this.getContractEvents(blockchainId, CONTRACTS.HUB_CONTRACT, currentBlock), ); } const contractEvents = await Promise.all(syncContractEventsPromises); @@ -137,7 +118,7 @@ class BlockchainEventListenerService { }, eventFetchInterval); } - async getContractEvents(blockchainId, contractName, currentBlock, eventsToFilter) { + async getContractEvents(blockchainId, contractName, currentBlock) { const lastCheckedBlockObject = await this.repositoryModuleManager.getLastCheckedBlock( blockchainId, contractName, @@ -146,7 +127,6 @@ class BlockchainEventListenerService { const events = await this.blockchainModuleManager.getAllPastEvents( blockchainId, contractName, - eventsToFilter, lastCheckedBlockObject?.lastCheckedBlock ?? 0, lastCheckedBlockObject?.lastCheckedTimestamp ?? 0, currentBlock, From 354b9888cddeaa1716ba4c1af9d51f8742e9dd88 Mon Sep 17 00:00:00 2001 From: zeroxbt <89495162+zeroxbt@users.noreply.github.com> Date: Wed, 14 Jun 2023 23:29:29 +0200 Subject: [PATCH 06/33] fix: do not skip protocol messages (#2578) * fix: do not skip protocol messages * add skip get request if operation completed --- .../common/protocol-message-command.js | 21 +++---------------- .../v1.0.0/v1-0-0-get-request-command.js | 16 ++++++++++++++ 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/src/commands/protocols/common/protocol-message-command.js b/src/commands/protocols/common/protocol-message-command.js index 5907dd055..0dbd4d6a8 100644 --- a/src/commands/protocols/common/protocol-message-command.js +++ b/src/commands/protocols/common/protocol-message-command.js @@ -1,9 +1,5 @@ import Command from '../../command.js'; -import { - NETWORK_MESSAGE_TYPES, - OPERATION_REQUEST_STATUS, - OPERATION_STATUS, -} from '../../../constants/constants.js'; +import { NETWORK_MESSAGE_TYPES, OPERATION_REQUEST_STATUS } from '../../../constants/constants.js'; class ProtocolMessageCommand extends Command { constructor(ctx) { @@ -20,19 +16,8 @@ class ProtocolMessageCommand extends Command { return this.sendProtocolMessage(command, message, messageType); } - async shouldSendMessage(command) { - const { operationId } = command.data; - - const { status } = await this.operationService.getOperationStatus(operationId); - - if (status === OPERATION_STATUS.IN_PROGRESS) { - return true; - } - this.logger.trace( - `${command.name} skipped for operationId: ${operationId} with status ${status}`, - ); - - return false; + async shouldSendMessage() { + return true; } async prepareMessage() { diff --git a/src/commands/protocols/get/sender/v1.0.0/v1-0-0-get-request-command.js b/src/commands/protocols/get/sender/v1.0.0/v1-0-0-get-request-command.js index 506644c95..c19163f32 100644 --- a/src/commands/protocols/get/sender/v1.0.0/v1-0-0-get-request-command.js +++ b/src/commands/protocols/get/sender/v1.0.0/v1-0-0-get-request-command.js @@ -3,6 +3,7 @@ import { NETWORK_MESSAGE_TIMEOUT_MILLS, ERROR_TYPE, OPERATION_REQUEST_STATUS, + OPERATION_STATUS, } from '../../../../../constants/constants.js'; class GetRequestCommand extends ProtocolRequestCommand { @@ -14,6 +15,21 @@ class GetRequestCommand extends ProtocolRequestCommand { this.errorType = ERROR_TYPE.GET.GET_REQUEST_ERROR; } + async shouldSendMessage(command) { + const { operationId } = command.data; + + const { status } = await this.operationService.getOperationStatus(operationId); + + if (status === OPERATION_STATUS.IN_PROGRESS) { + return true; + } + this.logger.trace( + `${command.name} skipped for operationId: ${operationId} with status ${status}`, + ); + + return false; + } + async prepareMessage(command) { const { assertionId, blockchain, contract, tokenId, hashFunctionId, state } = command.data; From 3a4fe317805e3cc60db8c31d7f75f4e936d57d2a Mon Sep 17 00:00:00 2001 From: zeroxbt <89495162+zeroxbt@users.noreply.github.com> Date: Wed, 14 Jun 2023 23:55:49 +0200 Subject: [PATCH 07/33] add epoch check telemetry event (#2564) * add epoch check telemetry event * add generate id function to operation id service --- .../protocols/common/epoch-check-command.js | 17 ++++++++++++++--- src/service/operation-id-service.js | 6 +++++- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/src/commands/protocols/common/epoch-check-command.js b/src/commands/protocols/common/epoch-check-command.js index f9e3730c9..3d674f498 100644 --- a/src/commands/protocols/common/epoch-check-command.js +++ b/src/commands/protocols/common/epoch-check-command.js @@ -1,10 +1,10 @@ /* eslint-disable no-await-in-loop */ -import { v4 as uuidv4 } from 'uuid'; import Command from '../../command.js'; import { COMMAND_QUEUE_PARALLELISM, COMMAND_RETRIES, TRANSACTION_CONFIRMATIONS, + OPERATION_ID_STATUS, } from '../../../constants/constants.js'; class EpochCheckCommand extends Command { @@ -19,6 +19,11 @@ class EpochCheckCommand extends Command { } async execute(command) { + const operationId = this.operationIdService.generateId(); + this.operationIdService.emitChangeEvent( + OPERATION_ID_STATUS.COMMIT_PROOF.EPOCH_CHECK_START, + operationId, + ); await Promise.all( this.blockchainModuleManager.getImplementationNames().map(async (blockchain) => { const commitWindowDurationPerc = @@ -65,6 +70,12 @@ class EpochCheckCommand extends Command { ]); }), ); + + this.operationIdService.emitChangeEvent( + OPERATION_ID_STATUS.COMMIT_PROOF.EPOCH_CHECK_END, + operationId, + ); + return Command.repeat(); } @@ -234,7 +245,7 @@ class EpochCheckCommand extends Command { async scheduleSubmitCommitCommand(agreement) { const commandData = { - operationId: uuidv4(), + operationId: this.operationIdService.generateId(), blockchain: agreement.blockchainId, contract: agreement.assetStorageContractAddress, tokenId: agreement.tokenId, @@ -256,7 +267,7 @@ class EpochCheckCommand extends Command { async scheduleSubmitProofsCommand(agreement) { const commandData = { - operationId: uuidv4(), + operationId: this.operationIdService.generateId(), blockchain: agreement.blockchainId, contract: agreement.assetStorageContractAddress, tokenId: agreement.tokenId, diff --git a/src/service/operation-id-service.js b/src/service/operation-id-service.js index eb727cbe7..40ead619f 100644 --- a/src/service/operation-id-service.js +++ b/src/service/operation-id-service.js @@ -1,4 +1,4 @@ -import { validate } from 'uuid'; +import { validate, v4 as uuidv4 } from 'uuid'; import path from 'path'; class OperationIdService { @@ -11,6 +11,10 @@ class OperationIdService { this.memoryCachedHandlersData = {}; } + generateId() { + return uuidv4(); + } + async generateOperationId(status) { const operationIdObject = await this.repositoryModuleManager.createOperationIdRecord({ status, From bf38cd3ec674740faaad12196e2f80b22fb1132d Mon Sep 17 00:00:00 2001 From: zeroxbt <89495162+zeroxbt@users.noreply.github.com> Date: Thu, 15 Jun 2023 09:51:50 +0200 Subject: [PATCH 08/33] only fetch necessary events (#2566) --- src/constants/constants.js | 32 ++++------------ .../blockchain/blockchain-module-manager.js | 2 + .../blockchain/implementation/web3-service.js | 21 ++++++++-- .../blockchain-event-listener-service.js | 38 ++++++++++++++----- 4 files changed, 56 insertions(+), 37 deletions(-) diff --git a/src/constants/constants.js b/src/constants/constants.js index 611ded831..0a142062e 100644 --- a/src/constants/constants.js +++ b/src/constants/constants.js @@ -27,7 +27,7 @@ export const TRIPLE_STORE_CONNECT_MAX_RETRIES = 10; export const DEFAULT_BLOCKCHAIN_EVENT_SYNC_PERIOD_IN_MILLS = 15 * 24 * 60 * 60 * 1000; // 15 days -export const MAXIMUM_NUMBERS_OF_BLOCKS_TO_FETCH = 500; +export const MAXIMUM_NUMBERS_OF_BLOCKS_TO_FETCH = 50; export const TRANSACTION_QUEUE_CONCURRENCY = 1; @@ -415,30 +415,12 @@ export const CONTRACTS = { }; export const CONTRACT_EVENTS = { - HUB: { - NEW_CONTRACT: 'NewContract', - CONTRACT_CHANGED: 'ContractChanged', - NEW_ASSET_STORAGE: 'NewAssetStorage', - ASSET_STORAGE_CHANGED: 'AssetStorageChanged', - }, - SHARDING_TABLE: { - NODE_ADDED: 'NodeAdded', - NODE_REMOVED: 'NodeRemoved', - }, - STAKING: { - STAKE_INCREASED: 'StakeIncreased', - STAKE_WITHDRAWAL_STARTED: 'StakeWithdrawalStarted', - }, - PROFILE: { - ASK_UPDATED: 'AskUpdated', - }, - COMMIT_MANAGER_V1: { - STATE_FINALIZED: 'StateFinalized', - }, - SERVICE_AGREEMENT_V1: { - SERVICE_AGREEMENT_V1_EXTENDED: 'ServiceAgreementV1Extended', - SERVICE_AGREEMENT_V1_TERMINATED: 'ServiceAgreementV1Terminated', - }, + HUB: ['NewContract', 'ContractChanged', 'NewAssetStorage', 'AssetStorageChanged'], + SHARDING_TABLE: ['NodeAdded', 'NodeRemoved'], + STAKING: ['StakeIncreased', 'StakeWithdrawalStarted'], + PROFILE: ['AskUpdated'], + COMMIT_MANAGER_V1: ['StateFinalized'], + SERVICE_AGREEMENT_V1: ['ServiceAgreementV1Extended', 'ServiceAgreementV1Terminated'], }; export const NODE_ENVIRONMENTS = { diff --git a/src/modules/blockchain/blockchain-module-manager.js b/src/modules/blockchain/blockchain-module-manager.js index 2079c1327..53c32c0d9 100644 --- a/src/modules/blockchain/blockchain-module-manager.js +++ b/src/modules/blockchain/blockchain-module-manager.js @@ -141,6 +141,7 @@ class BlockchainModuleManager extends BaseModuleManager { async getAllPastEvents( blockchain, contractName, + eventsToFilter, lastCheckedBlock, lastCheckedTimestamp, currentBlock, @@ -148,6 +149,7 @@ class BlockchainModuleManager extends BaseModuleManager { return this.callImplementationFunction(blockchain, 'getAllPastEvents', [ blockchain, contractName, + eventsToFilter, lastCheckedBlock, lastCheckedTimestamp, currentBlock, diff --git a/src/modules/blockchain/implementation/web3-service.js b/src/modules/blockchain/implementation/web3-service.js index 54f122b55..946a56202 100644 --- a/src/modules/blockchain/implementation/web3-service.js +++ b/src/modules/blockchain/implementation/web3-service.js @@ -396,6 +396,7 @@ class Web3Service { async getAllPastEvents( blockchainId, contractName, + eventsToFilter, lastCheckedBlock, lastCheckedTimestamp, currentBlock, @@ -412,14 +413,21 @@ class Web3Service { fromBlock = lastCheckedBlock + 1; } - let events = []; + const topics = []; + for (const filterName in contract.filters) { + if (!eventsToFilter.includes(filterName)) continue; + const filter = contract.filters[filterName]().topics[0]; + topics.push(filter); + } + + const events = []; while (fromBlock <= currentBlock) { const toBlock = Math.min( fromBlock + MAXIMUM_NUMBERS_OF_BLOCKS_TO_FETCH - 1, currentBlock, ); - const newEvents = await contract.queryFilter('*', fromBlock, toBlock); - events = events.concat(newEvents); + const newEvents = await this.processBlockRange(fromBlock, toBlock, contract, topics); + newEvents.forEach((e) => events.push(...e)); fromBlock = toBlock + 1; } @@ -439,6 +447,13 @@ class Web3Service { })); } + async processBlockRange(fromBlock, toBlock, contract, topics) { + const newEvents = await Promise.all( + topics.map((topic) => contract.queryFilter(topic, fromBlock, toBlock)), + ); + return newEvents; + } + isOlderThan(timestamp, olderThanInMills) { if (!timestamp) return true; const timestampThirtyDaysInPast = new Date().getTime() - olderThanInMills; diff --git a/src/service/blockchain-event-listener-service.js b/src/service/blockchain-event-listener-service.js index 2ce399559..6c3c0721c 100644 --- a/src/service/blockchain-event-listener-service.js +++ b/src/service/blockchain-event-listener-service.js @@ -11,10 +11,7 @@ import { const MAXIMUM_FETCH_EVENTS_FAILED_COUNT = 5; const fetchEventsFailedCount = {}; -const eventNames = []; -Object.keys(CONTRACT_EVENTS).forEach((contractName) => { - eventNames.push(...Object.values(CONTRACT_EVENTS[contractName])); -}); +const eventNames = Object.values(CONTRACT_EVENTS).flatMap((e) => e); class BlockchainEventListenerService { constructor(ctx) { @@ -52,24 +49,46 @@ class BlockchainEventListenerService { const currentBlock = await this.blockchainModuleManager.getBlockNumber(); const syncContractEventsPromises = [ - this.getContractEvents(blockchainId, CONTRACTS.SHARDING_TABLE_CONTRACT, currentBlock), - this.getContractEvents(blockchainId, CONTRACTS.STAKING_CONTRACT, currentBlock), - this.getContractEvents(blockchainId, CONTRACTS.PROFILE_CONTRACT, currentBlock), + this.getContractEvents( + blockchainId, + CONTRACTS.SHARDING_TABLE_CONTRACT, + currentBlock, + CONTRACT_EVENTS.SHARDING_TABLE, + ), + this.getContractEvents( + blockchainId, + CONTRACTS.STAKING_CONTRACT, + currentBlock, + CONTRACT_EVENTS.STAKING, + ), + this.getContractEvents( + blockchainId, + CONTRACTS.PROFILE_CONTRACT, + currentBlock, + CONTRACT_EVENTS.PROFILE, + ), this.getContractEvents( blockchainId, CONTRACTS.COMMIT_MANAGER_V1_U1_CONTRACT, currentBlock, + CONTRACT_EVENTS.COMMIT_MANAGER_V1, ), this.getContractEvents( blockchainId, CONTRACTS.SERVICE_AGREEMENT_V1_CONTRACT, currentBlock, + CONTRACT_EVENTS.SERVICE_AGREEMENT_V1, ), ]; if (!devEnvironment) { syncContractEventsPromises.push( - this.getContractEvents(blockchainId, CONTRACTS.HUB_CONTRACT, currentBlock), + this.getContractEvents( + blockchainId, + CONTRACTS.HUB_CONTRACT, + currentBlock, + CONTRACT_EVENTS.HUB, + ), ); } const contractEvents = await Promise.all(syncContractEventsPromises); @@ -118,7 +137,7 @@ class BlockchainEventListenerService { }, eventFetchInterval); } - async getContractEvents(blockchainId, contractName, currentBlock) { + async getContractEvents(blockchainId, contractName, currentBlock, eventsToFilter) { const lastCheckedBlockObject = await this.repositoryModuleManager.getLastCheckedBlock( blockchainId, contractName, @@ -127,6 +146,7 @@ class BlockchainEventListenerService { const events = await this.blockchainModuleManager.getAllPastEvents( blockchainId, contractName, + eventsToFilter, lastCheckedBlockObject?.lastCheckedBlock ?? 0, lastCheckedBlockObject?.lastCheckedTimestamp ?? 0, currentBlock, From 4e00d4069ffafcc5d16f5587cf00fd2df77047f3 Mon Sep 17 00:00:00 2001 From: zeroxbt <89495162+zeroxbt@users.noreply.github.com> Date: Thu, 15 Jun 2023 10:00:43 +0200 Subject: [PATCH 09/33] remove cached network sessions (#2575) * remove cached network sessions * fix remove cached network sessions * remove operationId key from sessions * remove network session on error * remove unused removeSession function --- .../common/handle-protocol-message-command.js | 3 + .../common/protocol-message-command.js | 11 +- .../network/implementation/libp2p-service.js | 211 ++++++++---------- src/modules/network/network-module-manager.js | 28 ++- 4 files changed, 120 insertions(+), 133 deletions(-) diff --git a/src/commands/protocols/common/handle-protocol-message-command.js b/src/commands/protocols/common/handle-protocol-message-command.js index 36e560dcc..43b74e4e8 100644 --- a/src/commands/protocols/common/handle-protocol-message-command.js +++ b/src/commands/protocols/common/handle-protocol-message-command.js @@ -38,6 +38,8 @@ class HandleProtocolMessageCommand extends Command { await this.handleError(error.message, command); } + this.networkModuleManager.removeCachedSession(operationId, keywordUuid, remotePeerId); + return Command.empty(); } @@ -223,6 +225,7 @@ class HandleProtocolMessageCommand extends Command { keywordUuid, { errorMessage }, ); + this.networkModuleManager.removeCachedSession(operationId, keywordUuid, remotePeerId); } } diff --git a/src/commands/protocols/common/protocol-message-command.js b/src/commands/protocols/common/protocol-message-command.js index 0dbd4d6a8..7e3c2a042 100644 --- a/src/commands/protocols/common/protocol-message-command.js +++ b/src/commands/protocols/common/protocol-message-command.js @@ -1,3 +1,4 @@ +import { v5 as uuidv5 } from 'uuid'; import Command from '../../command.js'; import { NETWORK_MESSAGE_TYPES, OPERATION_REQUEST_STATUS } from '../../../constants/constants.js'; @@ -27,16 +28,20 @@ class ProtocolMessageCommand extends Command { async sendProtocolMessage(command, message, messageType) { const { node, operationId, keyword } = command.data; + const keywordUuid = uuidv5(keyword, uuidv5.URL); + const response = await this.networkModuleManager.sendMessage( node.protocol, node.id, messageType, operationId, - keyword, + keywordUuid, message, this.messageTimeout(), ); + this.networkModuleManager.removeCachedSession(operationId, keywordUuid, node.id); + switch (response.header.messageType) { case NETWORK_MESSAGE_TYPES.RESPONSES.BUSY: return this.handleBusy(command, response.data); @@ -74,6 +79,10 @@ class ProtocolMessageCommand extends Command { } async recover(command, err) { + const { node, operationId, keyword } = command.data; + const keywordUuid = uuidv5(keyword, uuidv5.URL); + this.networkModuleManager.removeCachedSession(operationId, keywordUuid, node.id); + await this.markResponseAsFailed(command, err.message); return Command.empty(); } diff --git a/src/modules/network/implementation/libp2p-service.js b/src/modules/network/implementation/libp2p-service.js index 22f604a91..1b99a1c87 100644 --- a/src/modules/network/implementation/libp2p-service.js +++ b/src/modules/network/implementation/libp2p-service.js @@ -12,7 +12,6 @@ import { encode, decode } from 'it-length-prefixed'; import { create as _create, createFromPrivKey, createFromB58String } from 'peer-id'; import { InMemoryRateLimiter } from 'rolling-rate-limiter'; import toobusy from 'toobusy-js'; -import { v5 as uuidv5 } from 'uuid'; import { mkdir, writeFile, readFile, stat } from 'fs/promises'; import ip from 'ip'; import { TimeoutController } from 'timeout-abort-controller'; @@ -176,12 +175,12 @@ class Libp2pService { return this.node.multiaddrs; } - getProtocols(peerId) { - return this.node.peerStore.protoBook.get(peerId); + getProtocols(peerIdObject) { + return this.node.peerStore.protoBook.get(peerIdObject); } - getAddresses(peerId) { - return this.node.peerStore.addressBook.get(peerId); + getAddresses(peerIdObject) { + return this.node.peerStore.addressBook.get(peerIdObject); } getPeers() { @@ -197,82 +196,92 @@ class Libp2pService { this.node.handle(protocol, async (handlerProps) => { const { stream } = handlerProps; - const remotePeerId = handlerProps.connection.remotePeer.toB58String(); + const peerIdString = handlerProps.connection.remotePeer.toB58String(); const { message, valid, busy } = await this._readMessageFromStream( stream, this.isRequestValid.bind(this), - remotePeerId, + peerIdString, ); this.updateSessionStream( message.header.operationId, message.header.keywordUuid, - remotePeerId, + peerIdString, stream, ); if (!valid) { await this.sendMessageResponse( protocol, - remotePeerId, + peerIdString, NETWORK_MESSAGE_TYPES.RESPONSES.NACK, message.header.operationId, message.header.keywordUuid, { errorMessage: 'Invalid request message' }, ); + this.removeCachedSession( + message.header.operationId, + message.header.keywordUuid, + peerIdString, + ); } else if (busy) { await this.sendMessageResponse( protocol, - remotePeerId, + peerIdString, NETWORK_MESSAGE_TYPES.RESPONSES.BUSY, message.header.operationId, message.header.keywordUuid, {}, ); + this.removeCachedSession( + message.header.operationId, + message.header.keywordUuid, + peerIdString, + ); } else { this.logger.debug( - `Receiving message from ${remotePeerId} to ${this.config.id}: protocol: ${protocol}, messageType: ${message.header.messageType};`, + `Receiving message from ${peerIdString} to ${this.config.id}: protocol: ${protocol}, messageType: ${message.header.messageType};`, ); - await handler(message, remotePeerId); + await handler(message, peerIdString); } }); } - updateSessionStream(operationId, keywordUuid, remotePeerId, stream) { + updateSessionStream(operationId, keywordUuid, peerIdString, stream) { this.logger.trace( - `Storing new session stream for remotePeerId: ${remotePeerId} with operation id: ${operationId}`, + `Storing new session stream for remotePeerId: ${peerIdString} with operation id: ${operationId}`, ); - if (!this.sessions[remotePeerId]) { - this.sessions[remotePeerId] = { + if (!this.sessions[peerIdString]) { + this.sessions[peerIdString] = { [operationId]: { [keywordUuid]: { stream, }, }, }; - } else if (!this.sessions[remotePeerId][operationId]) { - this.sessions[remotePeerId][operationId] = { + } else if (!this.sessions[peerIdString][operationId]) { + this.sessions[peerIdString][operationId] = { [keywordUuid]: { stream, }, }; } else { - this.sessions[remotePeerId][operationId][keywordUuid] = { + this.sessions[peerIdString][operationId][keywordUuid] = { stream, }; } } - getSessionStream(operationId, keywordUuid, remotePeerId) { + getSessionStream(operationId, keywordUuid, peerIdString) { if ( - this.sessions[remotePeerId] && - this.sessions[remotePeerId][operationId] && - this.sessions[remotePeerId][operationId][keywordUuid] + this.sessions[peerIdString] && + this.sessions[peerIdString][operationId] && + this.sessions[peerIdString][operationId][keywordUuid] ) { this.logger.trace( - `Session found remotePeerId: ${remotePeerId}, operation id: ${operationId}`, + `Session found remotePeerId: ${peerIdString}, operation id: ${operationId}`, ); - return this.sessions[remotePeerId][operationId][keywordUuid].stream; + return this.sessions[peerIdString][operationId][keywordUuid].stream; } return null; } @@ -288,55 +297,57 @@ class Libp2pService { }; } - async sendMessage(protocol, peerId, messageType, operationId, keyword, message, timeout) { + async sendMessage( + protocol, + peerIdString, + messageType, + operationId, + keywordUuid, + message, + timeout, + ) { const nackMessage = { header: { messageType: NETWORK_MESSAGE_TYPES.RESPONSES.NACK }, data: { errorMessage: '', }, }; - const keywordUuid = uuidv5(keyword, uuidv5.URL); - - // const sessionStream = this.getSessionStream(operationId, remotePeerId.toB58String()); - // if (!sessionStream) { - // } else { - // stream = sessionStream; - // } - const remotePeerId = createFromB58String(peerId); + const peerIdObject = createFromB58String(peerIdString); - const publicIp = (this.getAddresses(remotePeerId) ?? []) + const publicIp = (this.getAddresses(peerIdObject) ?? []) .map((addr) => addr.multiaddr) .filter((addr) => addr.isThinWaistAddress()) .map((addr) => addr.toString().split('/')) .filter((splittedAddr) => !ip.isPrivate(splittedAddr[2]))[0]?.[2]; this.logger.trace( - `Dialing remotePeerId: ${remotePeerId.toB58String()} with public ip: ${publicIp}: protocol: ${protocol}, messageType: ${messageType} , operationId: ${operationId}`, + `Dialing remotePeerId: ${peerIdString} with public ip: ${publicIp}: protocol: ${protocol}, messageType: ${messageType} , operationId: ${operationId}`, ); let dialResult; let dialStart; let dialEnd; try { dialStart = Date.now(); - dialResult = await this.node.dialProtocol(remotePeerId, protocol); + dialResult = await this.node.dialProtocol(peerIdObject, protocol); dialEnd = Date.now(); } catch (error) { dialEnd = Date.now(); - nackMessage.data.errorMessage = `Unable to dial peer: ${remotePeerId.toB58String()}. protocol: ${protocol}, messageType: ${messageType} , operationId: ${operationId}, dial execution time: ${ + nackMessage.data.errorMessage = `Unable to dial peer: ${peerIdString}. protocol: ${protocol}, messageType: ${messageType} , operationId: ${operationId}, dial execution time: ${ dialEnd - dialStart } ms. Error: ${error.message}`; return nackMessage; } this.logger.trace( - `Created stream for peer: ${remotePeerId.toB58String()}. protocol: ${protocol}, messageType: ${messageType} , operationId: ${operationId}, dial execution time: ${ + `Created stream for peer: ${peerIdString}. protocol: ${protocol}, messageType: ${messageType} , operationId: ${operationId}, dial execution time: ${ dialEnd - dialStart } ms.`, ); + const { stream } = dialResult; - this.updateSessionStream(operationId, keywordUuid, remotePeerId.toB58String(), stream); + this.updateSessionStream(operationId, keywordUuid, peerIdString, stream); const streamMessage = this.createStreamMessage( message, @@ -346,7 +357,7 @@ class Libp2pService { ); this.logger.trace( - `Sending message to ${remotePeerId.toB58String()}. protocol: ${protocol}, messageType: ${messageType}, operationId: ${operationId}`, + `Sending message to ${peerIdString}. protocol: ${protocol}, messageType: ${messageType}, operationId: ${operationId}`, ); let sendMessageStart; @@ -357,27 +368,13 @@ class Libp2pService { sendMessageEnd = Date.now(); } catch (error) { sendMessageEnd = Date.now(); - nackMessage.data.errorMessage = `Unable to send message to peer: ${remotePeerId.toB58String()}. protocol: ${protocol}, messageType: ${messageType}, operationId: ${operationId}, execution time: ${ + nackMessage.data.errorMessage = `Unable to send message to peer: ${peerIdString}. protocol: ${protocol}, messageType: ${messageType}, operationId: ${operationId}, execution time: ${ sendMessageEnd - sendMessageStart } ms. Error: ${error.message}`; return nackMessage; } - // if (!this.sessions[remotePeerId.toB58String()]) { - // this.sessions[remotePeerId.toB58String()] = { - // [operationId]: { - // stream - // } - // } - // } else { - // this.sessions[remotePeerId.toB58String()][operationId] = { - // stream - // } - // } - // if (!this.sessions.sender[message.header.sessionId]) { - // this.sessions.sender[message.header.sessionId] = {}; - // } let readResponseStart; let readResponseEnd; let response; @@ -397,7 +394,7 @@ class Libp2pService { response = await this._readMessageFromStream( stream, this.isResponseValid.bind(this), - remotePeerId.toB58String(), + peerIdString, ); if (timeoutController.signal.aborted) { @@ -413,7 +410,7 @@ class Libp2pService { timeoutController.clear(); readResponseEnd = Date.now(); - nackMessage.data.errorMessage = `Unable to read response from peer ${remotePeerId.toB58String()}. protocol: ${protocol}, messageType: ${messageType} , operationId: ${operationId}, execution time: ${ + nackMessage.data.errorMessage = `Unable to read response from peer ${peerIdString}. protocol: ${protocol}, messageType: ${messageType} , operationId: ${operationId}, execution time: ${ readResponseEnd - readResponseStart } ms. Error: ${error.message}`; @@ -421,7 +418,7 @@ class Libp2pService { } this.logger.trace( - `Receiving response from ${remotePeerId.toB58String()}. protocol: ${protocol}, messageType: ${ + `Receiving response from ${peerIdString}. protocol: ${protocol}, messageType: ${ response.message?.header?.messageType }, operationId: ${operationId}, execution time: ${ readResponseEnd - readResponseStart @@ -439,19 +436,19 @@ class Libp2pService { async sendMessageResponse( protocol, - remotePeerId, + peerIdString, messageType, operationId, keywordUuid, message, ) { this.logger.debug( - `Sending response from ${this.config.id} to ${remotePeerId}: protocol: ${protocol}, messageType: ${messageType};`, + `Sending response from ${this.config.id} to ${peerIdString}: protocol: ${protocol}, messageType: ${messageType};`, ); - const stream = this.getSessionStream(operationId, keywordUuid, remotePeerId); + const stream = this.getSessionStream(operationId, keywordUuid, peerIdString); if (!stream) { - throw Error(`Unable to find opened stream for remotePeerId: ${remotePeerId}`); + throw Error(`Unable to find opened stream for remotePeerId: ${peerIdString}`); } const response = this.createStreamMessage(message, operationId, keywordUuid, messageType); @@ -459,35 +456,6 @@ class Libp2pService { await this._sendMessageToStream(stream, response); } - // updateReceiverSession(header) { - // // if BUSY we expect same request, so don't update session - // if (header.messageType === constants.NETWORK_MESSAGE_TYPES.RESPONSES.BUSY) return; - // // if NACK we don't expect other requests, so delete session - // if (header.messageType === constants.NETWORK_MESSAGE_TYPES.RESPONSES.NACK) { - // if (header.sessionId) delete this.sessions.receiver[header.sessionId]; - // return; - // } - // - // // if session is new, initialise array of expected message types - // if (!this.sessions.receiver[header.sessionId].expectedMessageTypes) { - // this.sessions.receiver[header.sessionId].expectedMessageTypes = Object.keys( - // constants.NETWORK_MESSAGE_TYPES.REQUESTS, - // ); - // } - // - // // subroutine completed - // if (header.messageType === constants.NETWORK_MESSAGE_TYPES.RESPONSES.ACK) { - // // protocol operation completed, delete session - // if (this.sessions.receiver[header.sessionId].expectedMessageTypes.length <= 1) { - // this.removeSession(header.sessionId); - // } else { - // // operation not completed, update array of expected message types - // this.sessions.receiver[header.sessionId].expectedMessageTypes = - // this.sessions.receiver[header.sessionId].expectedMessageTypes.slice(1); - // } - // } - // } - async _sendMessageToStream(stream, message) { const stringifiedHeader = JSON.stringify(message.header); const stringifiedData = JSON.stringify(message.data); @@ -511,7 +479,7 @@ class Libp2pService { ); } - async _readMessageFromStream(stream, isMessageValid, remotePeerId) { + async _readMessageFromStream(stream, isMessageValid, peerIdString) { return pipe( // Read from the stream (the source) stream.source, @@ -520,11 +488,11 @@ class Libp2pService { // Turn buffers into strings (source) => map(source, (buf) => buf.toString()), // Sink function - (source) => this.readMessageSink(source, isMessageValid, remotePeerId), + (source) => this.readMessageSink(source, isMessageValid, peerIdString), ); } - async readMessageSink(source, isMessageValid, remotePeerId) { + async readMessageSink(source, isMessageValid, peerIdString) { const message = { header: { operationId: '', keywordUuid: '' }, data: {} }; // we expect first buffer to be header const stringifiedHeader = (await source.next()).value; @@ -536,7 +504,7 @@ class Libp2pService { message.header = JSON.parse(stringifiedHeader); // validate request / response - if (!(await isMessageValid(message.header, remotePeerId))) { + if (!(await isMessageValid(message.header, peerIdString))) { return { message, valid: false }; } @@ -558,9 +526,9 @@ class Libp2pService { return { message, valid: true, busy: false }; } - async isRequestValid(header, remotePeerId) { + async isRequestValid(header, peerIdString) { // filter spam requests - if (await this.limitRequest(header, remotePeerId)) return false; + if (await this.limitRequest(header, peerIdString)) return false; // header well formed if ( @@ -574,30 +542,15 @@ class Libp2pService { return true; } - return this.sessionExists(remotePeerId, header.operationId, header.keywordUuid); + return this.sessionExists(peerIdString, header.operationId, header.keywordUuid); } sessionExists() { return true; - // return this.sessions[remotePeerId.toB58String()] && this.sessions[remotePeerId.toB58String()][operationId]; } async isResponseValid() { return true; - // return ( - // header.operationId && - // header.messageType && - // this.sessions[remotePeerId][header.operationId] && - // Object.keys(constants.NETWORK_MESSAGE_TYPES.RESPONSES).includes(header.messageType) - // ); - } - - removeSession(sessionId) { - if (this.sessions.sender[sessionId]) { - delete this.sessions.sender[sessionId]; - } else if (this.sessions.receiver[sessionId]) { - delete this.sessions.receiver[sessionId]; - } } healthCheck() { @@ -607,36 +560,36 @@ class Libp2pService { return false; } - async limitRequest(header, remotePeerId) { + async limitRequest(header, peerIdString) { // if (header.sessionId && this.sessions.receiver[header.sessionId]) return false; - if (this.blackList[remotePeerId]) { + if (this.blackList[peerIdString]) { const remainingMinutes = Math.floor( NETWORK_API_BLACK_LIST_TIME_WINDOW_MINUTES - - (Date.now() - this.blackList[remotePeerId]) / (1000 * 60), + (Date.now() - this.blackList[peerIdString]) / (1000 * 60), ); if (remainingMinutes > 0) { this.logger.debug( - `Blocking request from ${remotePeerId}. Node is blacklisted for ${remainingMinutes} minutes.`, + `Blocking request from ${peerIdString}. Node is blacklisted for ${remainingMinutes} minutes.`, ); return true; } - delete this.blackList[remotePeerId]; + delete this.blackList[peerIdString]; } - if (await this.rateLimiter.spamDetection.limit(remotePeerId)) { - this.blackList[remotePeerId] = Date.now(); + if (await this.rateLimiter.spamDetection.limit(peerIdString)) { + this.blackList[peerIdString] = Date.now(); this.logger.debug( - `Blocking request from ${remotePeerId}. Spammer detected and blacklisted for ${NETWORK_API_BLACK_LIST_TIME_WINDOW_MINUTES} minutes.`, + `Blocking request from ${peerIdString}. Spammer detected and blacklisted for ${NETWORK_API_BLACK_LIST_TIME_WINDOW_MINUTES} minutes.`, ); return true; } - if (await this.rateLimiter.basicRateLimiter.limit(remotePeerId)) { + if (await this.rateLimiter.basicRateLimiter.limit(peerIdString)) { this.logger.debug( - `Blocking request from ${remotePeerId}. Max number of requests exceeded.`, + `Blocking request from ${peerIdString}. Max number of requests exceeded.`, ); return true; @@ -674,6 +627,16 @@ class Libp2pService { async getPeerInfo(peerId) { return this.node.peerStore.get(createFromB58String(peerId)); } + + removeCachedSession(operationId, keywordUuid, peerIdString) { + if (this.sessions[peerIdString]?.[operationId]?.[keywordUuid]?.stream) { + this.sessions[peerIdString][operationId][keywordUuid].stream.close(); + delete this.sessions[peerIdString][operationId]; + this.logger.trace( + `Removed session for remotePeerId: ${peerIdString}, operationId: ${operationId}.`, + ); + } + } } export default Libp2pService; diff --git a/src/modules/network/network-module-manager.js b/src/modules/network/network-module-manager.js index 6ba254d66..758ca4244 100644 --- a/src/modules/network/network-module-manager.js +++ b/src/modules/network/network-module-manager.js @@ -23,14 +23,22 @@ class NetworkModuleManager extends BaseModuleManager { } } - async sendMessage(protocol, remotePeerId, messageType, operationId, keyword, message, timeout) { + async sendMessage( + protocol, + remotePeerId, + messageType, + operationId, + keywordUuid, + message, + timeout, + ) { if (this.initialized) { return this.getImplementation().module.sendMessage( protocol, remotePeerId, messageType, operationId, - keyword, + keywordUuid, message, timeout, ); @@ -56,12 +64,6 @@ class NetworkModuleManager extends BaseModuleManager { } } - removeSession(sessionId) { - if (this.initialized) { - this.getImplementation().module.removeSession(sessionId); - } - } - getPeerId() { if (this.initialized) { return this.getImplementation().module.getPeerId(); @@ -91,6 +93,16 @@ class NetworkModuleManager extends BaseModuleManager { return this.getImplementation().module.getPeerInfo(peerId); } } + + removeCachedSession(operationId, keywordUuid, remotePeerId) { + if (this.initialized) { + this.getImplementation().module.removeCachedSession( + operationId, + keywordUuid, + remotePeerId, + ); + } + } } export default NetworkModuleManager; From 367b950ad55ed3012597afd32459a80de445ecc5 Mon Sep 17 00:00:00 2001 From: djordjekovac Date: Fri, 16 Jun 2023 11:05:19 +0200 Subject: [PATCH 10/33] Operational db data archive (#2579) * Added blockchain event cleaner command * Added operation response cleanup commands * Updated cleaner commands --- .../blockchain-event-cleaner-command.js | 49 ++++++++++++ src/commands/cleaners/cleaner-command.js | 76 +++++++++++++++++++ .../cleaners/commands-cleaner-command.js | 48 ++++++++++++ .../cleaners/get-response-cleaner-command.js | 51 +++++++++++++ .../operation-id-cleaner-command.js | 0 .../publish-response-cleaner-command.js | 51 +++++++++++++ .../update-response-cleaner-command.js | 51 +++++++++++++ .../common/commands-cleaner-command.js | 57 -------------- src/constants/constants.js | 34 ++++++++- .../sequelize/models/blockchain-event.js | 2 + .../blockchain-event-repository.js | 20 +++++ .../repositories/command-repository.js | 25 +++++- .../repositories/operation-response.js | 21 +++++ .../repository/repository-module-manager.js | 28 ++++++- src/service/archive-service.js | 19 +++++ src/service/file-service.js | 6 ++ 16 files changed, 475 insertions(+), 63 deletions(-) create mode 100644 src/commands/cleaners/blockchain-event-cleaner-command.js create mode 100644 src/commands/cleaners/cleaner-command.js create mode 100644 src/commands/cleaners/commands-cleaner-command.js create mode 100644 src/commands/cleaners/get-response-cleaner-command.js rename src/commands/{common => cleaners}/operation-id-cleaner-command.js (100%) create mode 100644 src/commands/cleaners/publish-response-cleaner-command.js create mode 100644 src/commands/cleaners/update-response-cleaner-command.js delete mode 100644 src/commands/common/commands-cleaner-command.js create mode 100644 src/service/archive-service.js diff --git a/src/commands/cleaners/blockchain-event-cleaner-command.js b/src/commands/cleaners/blockchain-event-cleaner-command.js new file mode 100644 index 000000000..9eda2c1f0 --- /dev/null +++ b/src/commands/cleaners/blockchain-event-cleaner-command.js @@ -0,0 +1,49 @@ +import { + PROCESSED_BLOCKCHAIN_EVENTS_CLEANUP_TIME_MILLS, + REPOSITORY_ROWS_FOR_REMOVAL_MAX_NUMBER, + PROCESSED_BLOCKCHAIN_EVENTS_CLEANUP_TIME_DELAY, + ARCHIVE_BLOCKCHAIN_EVENTS_FOLDER, +} from '../../constants/constants.js'; +import CleanerCommand from './cleaner-command.js'; + +class BlockchainEventCleanerCommand extends CleanerCommand { + constructor(ctx) { + super(ctx); + this.logger = ctx.logger; + this.repositoryModuleManager = ctx.repositoryModuleManager; + this.archiveService = ctx.archiveService; + } + + async findRowsForRemoval(nowTimestamp) { + return this.repositoryModuleManager.findProcessedEvents( + nowTimestamp - PROCESSED_BLOCKCHAIN_EVENTS_CLEANUP_TIME_DELAY, + REPOSITORY_ROWS_FOR_REMOVAL_MAX_NUMBER, + ); + } + + getArchiveFolderName() { + return ARCHIVE_BLOCKCHAIN_EVENTS_FOLDER; + } + + async deleteRows(ids) { + return this.repositoryModuleManager.removeEvents(ids); + } + + /** + * Builds default command + * @param map + * @returns {{add, data: *, delay: *, deadline: *}} + */ + default(map) { + const command = { + name: 'blockchainEventCleanerCommand', + data: {}, + period: PROCESSED_BLOCKCHAIN_EVENTS_CLEANUP_TIME_MILLS, + transactional: false, + }; + Object.assign(command, map); + return command; + } +} + +export default BlockchainEventCleanerCommand; diff --git a/src/commands/cleaners/cleaner-command.js b/src/commands/cleaners/cleaner-command.js new file mode 100644 index 000000000..c27cd00d0 --- /dev/null +++ b/src/commands/cleaners/cleaner-command.js @@ -0,0 +1,76 @@ +import { REPOSITORY_ROWS_FOR_REMOVAL_MAX_NUMBER } from '../../constants/constants.js'; +import Command from '../command.js'; + +class CleanerCommand extends Command { + constructor(ctx) { + super(ctx); + this.logger = ctx.logger; + this.repositoryModuleManager = ctx.repositoryModuleManager; + this.archiveService = ctx.archiveService; + } + + /** + * Executes command and produces one or more events + * @param command + */ + async execute() { + const nowTimestamp = Date.now(); + + let rowsForRemoval = await this.findRowsForRemoval(nowTimestamp); + + while (rowsForRemoval?.length >= REPOSITORY_ROWS_FOR_REMOVAL_MAX_NUMBER) { + const archiveName = this.getArchiveName(rowsForRemoval); + + // eslint-disable-next-line no-await-in-loop + await this.archiveService.archiveData( + this.getArchiveFolderName(), + archiveName, + rowsForRemoval, + ); + + // remove from database; + const ids = rowsForRemoval.map((command) => command.id); + // eslint-disable-next-line no-await-in-loop + await this.deleteRows(ids); + + // eslint-disable-next-line no-await-in-loop + rowsForRemoval = await this.findRowsForRemoval(nowTimestamp); + } + + return Command.repeat(); + } + + getArchiveName(rowsForRemoval) { + const firstTimestamp = new Date(rowsForRemoval[0].createdAt).getTime(); + const lastTimestamp = new Date( + rowsForRemoval[rowsForRemoval.length - 1].createdAt, + ).getTime(); + return `${firstTimestamp}-${lastTimestamp}.json`; + } + + // eslint-disable-next-line no-unused-vars + async findRowsForRemoval(nowTimestamp) { + throw Error('findRowsForRemoval not implemented'); + } + + getArchiveFolderName() { + throw Error('getArchiveFolderName not implemented'); + } + + // eslint-disable-next-line no-unused-vars + async deleteRows(ids) { + throw Error('deleteRows not implemented'); + } + + /** + * Recover system from failure + * @param command + * @param error + */ + async recover(command, error) { + this.logger.warn(`Failed to clean operational db data: error: ${error.message}`); + return Command.repeat(); + } +} + +export default CleanerCommand; diff --git a/src/commands/cleaners/commands-cleaner-command.js b/src/commands/cleaners/commands-cleaner-command.js new file mode 100644 index 000000000..94aa70683 --- /dev/null +++ b/src/commands/cleaners/commands-cleaner-command.js @@ -0,0 +1,48 @@ +import { + FINALIZED_COMMAND_CLEANUP_TIME_MILLS, + ARCHIVE_COMMANDS_FOLDER, + REPOSITORY_ROWS_FOR_REMOVAL_MAX_NUMBER, +} from '../../constants/constants.js'; +import CleanerCommand from './cleaner-command.js'; + +class CommandsCleanerCommand extends CleanerCommand { + constructor(ctx) { + super(ctx); + this.logger = ctx.logger; + this.repositoryModuleManager = ctx.repositoryModuleManager; + this.archiveService = ctx.archiveService; + } + + async findRowsForRemoval(nowTimestamp) { + return this.repositoryModuleManager.findFinalizedCommands( + nowTimestamp, + REPOSITORY_ROWS_FOR_REMOVAL_MAX_NUMBER, + ); + } + + getArchiveFolderName() { + return ARCHIVE_COMMANDS_FOLDER; + } + + async deleteRows(ids) { + return this.repositoryModuleManager.removeCommands(ids); + } + + /** + * Builds default command + * @param map + * @returns {{add, data: *, delay: *, deadline: *}} + */ + default(map) { + const command = { + name: 'commandsCleanerCommand', + data: {}, + period: FINALIZED_COMMAND_CLEANUP_TIME_MILLS, + transactional: false, + }; + Object.assign(command, map); + return command; + } +} + +export default CommandsCleanerCommand; diff --git a/src/commands/cleaners/get-response-cleaner-command.js b/src/commands/cleaners/get-response-cleaner-command.js new file mode 100644 index 000000000..ae035d634 --- /dev/null +++ b/src/commands/cleaners/get-response-cleaner-command.js @@ -0,0 +1,51 @@ +import CleanerCommand from './cleaner-command.js'; +import { + REPOSITORY_ROWS_FOR_REMOVAL_MAX_NUMBER, + OPERATIONS, + GET_RESPONSE_CLEANUP_TIME_DELAY, + GET_RESPONSE_CLEANUP_TIME_MILLS, + ARCHIVE_GET_RESPONSES_FOLDER, +} from '../../constants/constants.js'; + +class GetResponseCleanerCommand extends CleanerCommand { + constructor(ctx) { + super(ctx); + this.logger = ctx.logger; + this.repositoryModuleManager = ctx.repositoryModuleManager; + this.archiveService = ctx.archiveService; + } + + async findRowsForRemoval(nowTimestamp) { + return this.repositoryModuleManager.findProcessedOperationResponse( + nowTimestamp - GET_RESPONSE_CLEANUP_TIME_DELAY, + REPOSITORY_ROWS_FOR_REMOVAL_MAX_NUMBER, + OPERATIONS.GET, + ); + } + + getArchiveFolderName() { + return ARCHIVE_GET_RESPONSES_FOLDER; + } + + async deleteRows(ids) { + return this.repositoryModuleManager.removeOperationResponse(ids, OPERATIONS.GET); + } + + /** + * Builds default command + * @param map + * @returns {{add, data: *, delay: *, deadline: *}} + */ + default(map) { + const command = { + name: 'getResponseCleanerCommand', + data: {}, + period: GET_RESPONSE_CLEANUP_TIME_MILLS, + transactional: false, + }; + Object.assign(command, map); + return command; + } +} + +export default GetResponseCleanerCommand; diff --git a/src/commands/common/operation-id-cleaner-command.js b/src/commands/cleaners/operation-id-cleaner-command.js similarity index 100% rename from src/commands/common/operation-id-cleaner-command.js rename to src/commands/cleaners/operation-id-cleaner-command.js diff --git a/src/commands/cleaners/publish-response-cleaner-command.js b/src/commands/cleaners/publish-response-cleaner-command.js new file mode 100644 index 000000000..1a3c981be --- /dev/null +++ b/src/commands/cleaners/publish-response-cleaner-command.js @@ -0,0 +1,51 @@ +import CleanerCommand from './cleaner-command.js'; +import { + REPOSITORY_ROWS_FOR_REMOVAL_MAX_NUMBER, + OPERATIONS, + PUBLISH_RESPONSE_CLEANUP_TIME_DELAY, + PUBLISH_RESPONSE_CLEANUP_TIME_MILLS, + ARCHIVE_PUBLISH_RESPONSES_FOLDER, +} from '../../constants/constants.js'; + +class PublishResponseCleanerCommand extends CleanerCommand { + constructor(ctx) { + super(ctx); + this.logger = ctx.logger; + this.repositoryModuleManager = ctx.repositoryModuleManager; + this.archiveService = ctx.archiveService; + } + + async findRowsForRemoval(nowTimestamp) { + return this.repositoryModuleManager.findProcessedOperationResponse( + nowTimestamp - PUBLISH_RESPONSE_CLEANUP_TIME_DELAY, + REPOSITORY_ROWS_FOR_REMOVAL_MAX_NUMBER, + OPERATIONS.PUBLISH, + ); + } + + getArchiveFolderName() { + return ARCHIVE_PUBLISH_RESPONSES_FOLDER; + } + + async deleteRows(ids) { + return this.repositoryModuleManager.removeOperationResponse(ids, OPERATIONS.PUBLISH); + } + + /** + * Builds default command + * @param map + * @returns {{add, data: *, delay: *, deadline: *}} + */ + default(map) { + const command = { + name: 'publishResponseCleanerCommand', + data: {}, + period: PUBLISH_RESPONSE_CLEANUP_TIME_MILLS, + transactional: false, + }; + Object.assign(command, map); + return command; + } +} + +export default PublishResponseCleanerCommand; diff --git a/src/commands/cleaners/update-response-cleaner-command.js b/src/commands/cleaners/update-response-cleaner-command.js new file mode 100644 index 000000000..e0f54d9c9 --- /dev/null +++ b/src/commands/cleaners/update-response-cleaner-command.js @@ -0,0 +1,51 @@ +import CleanerCommand from './cleaner-command.js'; +import { + REPOSITORY_ROWS_FOR_REMOVAL_MAX_NUMBER, + OPERATIONS, + UPDATE_RESPONSE_CLEANUP_TIME_DELAY, + UPDATE_RESPONSE_CLEANUP_TIME_MILLS, + ARCHIVE_UPDATE_RESPONSES_FOLDER, +} from '../../constants/constants.js'; + +class UpdateResponseCleanerCommand extends CleanerCommand { + constructor(ctx) { + super(ctx); + this.logger = ctx.logger; + this.repositoryModuleManager = ctx.repositoryModuleManager; + this.archiveService = ctx.archiveService; + } + + async findRowsForRemoval(nowTimestamp) { + return this.repositoryModuleManager.findProcessedOperationResponse( + nowTimestamp - UPDATE_RESPONSE_CLEANUP_TIME_DELAY, + REPOSITORY_ROWS_FOR_REMOVAL_MAX_NUMBER, + OPERATIONS.UPDATE, + ); + } + + getArchiveFolderName() { + return ARCHIVE_UPDATE_RESPONSES_FOLDER; + } + + async deleteRows(ids) { + return this.repositoryModuleManager.removeOperationResponse(ids, OPERATIONS.UPDATE); + } + + /** + * Builds default command + * @param map + * @returns {{add, data: *, delay: *, deadline: *}} + */ + default(map) { + const command = { + name: 'updateResponseCleanerCommand', + data: {}, + period: UPDATE_RESPONSE_CLEANUP_TIME_MILLS, + transactional: false, + }; + Object.assign(command, map); + return command; + } +} + +export default UpdateResponseCleanerCommand; diff --git a/src/commands/common/commands-cleaner-command.js b/src/commands/common/commands-cleaner-command.js deleted file mode 100644 index a1f214eec..000000000 --- a/src/commands/common/commands-cleaner-command.js +++ /dev/null @@ -1,57 +0,0 @@ -import Command from '../command.js'; -// eslint-disable-next-line no-unused-vars -import { COMMAND_STATUS, FINALIZED_COMMAND_CLEANUP_TIME_MILLS } from '../../constants/constants.js'; - -/** - * Increases approval for Bidding contract on blockchain - */ -class CommandsCleanerCommand extends Command { - constructor(ctx) { - super(ctx); - this.logger = ctx.logger; - this.repositoryModuleManager = ctx.repositoryModuleManager; - } - - /** - * Executes command and produces one or more events - * @param command - */ - async execute() { - // TODO: Uncomment after discussion - // await this.repositoryModuleManager.removeFinalizedCommands([ - // COMMAND_STATUS.COMPLETED, - // COMMAND_STATUS.FAILED, - // COMMAND_STATUS.EXPIRED, - // COMMAND_STATUS.UNKNOWN, - // ]); - return Command.repeat(); - } - - /** - * Recover system from failure - * @param command - * @param error - */ - async recover(command, error) { - this.logger.warn(`Failed to clean finalized commands: error: ${error.message}`); - return Command.repeat(); - } - - /** - * Builds default command - * @param map - * @returns {{add, data: *, delay: *, deadline: *}} - */ - default(map) { - const command = { - name: 'commandsCleanerCommand', - data: {}, - period: FINALIZED_COMMAND_CLEANUP_TIME_MILLS, - transactional: false, - }; - Object.assign(command, map); - return command; - } -} - -export default CommandsCleanerCommand; diff --git a/src/constants/constants.js b/src/constants/constants.js index 0a142062e..0999842ab 100644 --- a/src/constants/constants.js +++ b/src/constants/constants.js @@ -130,6 +130,10 @@ export const PERMANENT_COMMANDS = [ 'commandsCleanerCommand', 'dialPeersCommand', 'epochCheckCommand', + 'blockchainEventCleanerCommand', + 'getResponseCleanerCommand', + 'publishResponseCleanerCommand', + 'updateResponseCleanerCommand', ]; export const MAX_COMMAND_DELAY_IN_MILLS = 14400 * 60 * 1000; // 10 days @@ -336,7 +340,23 @@ export const OPERATION_ID_COMMAND_CLEANUP_TIME_MILLS = 24 * 60 * 60 * 1000; * @constant {number} FINALIZED_COMMAND_CLEANUP_TIME_MILLS - Command cleanup interval time * finalized commands command cleanup interval time 24h */ -export const FINALIZED_COMMAND_CLEANUP_TIME_MILLS = 30 * 24 * 60 * 60 * 1000; +export const FINALIZED_COMMAND_CLEANUP_TIME_MILLS = 24 * 60 * 60 * 1000; + +export const GET_RESPONSE_CLEANUP_TIME_MILLS = 24 * 60 * 60 * 1000; + +export const GET_RESPONSE_CLEANUP_TIME_DELAY = 24 * 60 * 60 * 1000; + +export const PUBLISH_RESPONSE_CLEANUP_TIME_MILLS = 24 * 60 * 60 * 1000; + +export const PUBLISH_RESPONSE_CLEANUP_TIME_DELAY = 24 * 60 * 60 * 1000; + +export const UPDATE_RESPONSE_CLEANUP_TIME_MILLS = 24 * 60 * 60 * 1000; + +export const UPDATE_RESPONSE_CLEANUP_TIME_DELAY = 24 * 60 * 60 * 1000; + +export const PROCESSED_BLOCKCHAIN_EVENTS_CLEANUP_TIME_MILLS = 24 * 60 * 60 * 1000; + +export const PROCESSED_BLOCKCHAIN_EVENTS_CLEANUP_TIME_DELAY = 24 * 60 * 60 * 1000; /** * @constant {number} COMMAND_STATUS - * Status for commands @@ -351,6 +371,18 @@ export const COMMAND_STATUS = { REPEATING: 'REPEATING', }; +export const REPOSITORY_ROWS_FOR_REMOVAL_MAX_NUMBER = 1000; + +export const ARCHIVE_COMMANDS_FOLDER = 'commands'; + +export const ARCHIVE_BLOCKCHAIN_EVENTS_FOLDER = 'blockchain_events'; + +export const ARCHIVE_GET_RESPONSES_FOLDER = 'get_responses'; + +export const ARCHIVE_PUBLISH_RESPONSES_FOLDER = 'publish_responses'; + +export const ARCHIVE_UPDATE_RESPONSES_FOLDER = 'update_responses'; + /** * How many commands will run in parallel * @type {number} diff --git a/src/modules/repository/implementation/sequelize/models/blockchain-event.js b/src/modules/repository/implementation/sequelize/models/blockchain-event.js index c555dd1b8..b15537849 100644 --- a/src/modules/repository/implementation/sequelize/models/blockchain-event.js +++ b/src/modules/repository/implementation/sequelize/models/blockchain-event.js @@ -13,6 +13,8 @@ export default (sequelize, DataTypes) => { data: DataTypes.TEXT, block: DataTypes.INTEGER, processed: DataTypes.BOOLEAN, + createdAt: DataTypes.DATE, + updatedAt: DataTypes.DATE, }, { underscored: true }, ); diff --git a/src/modules/repository/implementation/sequelize/repositories/blockchain-event-repository.js b/src/modules/repository/implementation/sequelize/repositories/blockchain-event-repository.js index 9d21770b4..151f221dd 100644 --- a/src/modules/repository/implementation/sequelize/repositories/blockchain-event-repository.js +++ b/src/modules/repository/implementation/sequelize/repositories/blockchain-event-repository.js @@ -55,6 +55,26 @@ class BlockchainEventRepository { }, ); } + + async removeEvents(ids) { + await this.model.destroy({ + where: { + id: { [Sequelize.Op.in]: ids }, + }, + }); + } + + async findProcessedEvents(timestamp, limit) { + return this.model.findAll({ + where: { + processed: true, + createdAt: { [Sequelize.Op.lte]: timestamp }, + }, + order: [['createdAt', 'asc']], + raw: true, + limit, + }); + } } export default BlockchainEventRepository; diff --git a/src/modules/repository/implementation/sequelize/repositories/command-repository.js b/src/modules/repository/implementation/sequelize/repositories/command-repository.js index b272f1b71..44c08b6be 100644 --- a/src/modules/repository/implementation/sequelize/repositories/command-repository.js +++ b/src/modules/repository/implementation/sequelize/repositories/command-repository.js @@ -1,4 +1,5 @@ import Sequelize from 'sequelize'; +import { COMMAND_STATUS } from '../../../../../constants/constants.js'; class CommandRepository { constructor(models) { @@ -41,14 +42,32 @@ class CommandRepository { }); } - async removeFinalizedCommands(finalizedStatuses) { + async removeCommands(ids) { await this.model.destroy({ where: { - status: { [Sequelize.Op.in]: finalizedStatuses }, - startedAt: { [Sequelize.Op.lte]: Date.now() }, + id: { [Sequelize.Op.in]: ids }, }, }); } + + async findFinalizedCommands(timestamp, limit) { + return this.model.findAll({ + where: { + status: { + [Sequelize.Op.in]: [ + COMMAND_STATUS.COMPLETED, + COMMAND_STATUS.FAILED, + COMMAND_STATUS.EXPIRED, + COMMAND_STATUS.UNKNOWN, + ], + }, + startedAt: { [Sequelize.Op.lte]: timestamp }, + }, + order: [['startedAt', 'asc']], + raw: true, + limit, + }); + } } export default CommandRepository; diff --git a/src/modules/repository/implementation/sequelize/repositories/operation-response.js b/src/modules/repository/implementation/sequelize/repositories/operation-response.js index 87854a2b2..7c55f5d13 100644 --- a/src/modules/repository/implementation/sequelize/repositories/operation-response.js +++ b/src/modules/repository/implementation/sequelize/repositories/operation-response.js @@ -1,3 +1,5 @@ +import Sequelize from 'sequelize'; + class OperationResponseRepository { constructor(models) { this.sequelize = models.sequelize; @@ -25,6 +27,25 @@ class OperationResponseRepository { }, }); } + + async findProcessedOperationResponse(timestamp, limit, operation) { + return this.models[`${operation}_response`].findAll({ + where: { + createdAt: { [Sequelize.Op.lte]: timestamp }, + }, + order: [['createdAt', 'asc']], + raw: true, + limit, + }); + } + + async removeOperationResponse(ids, operation) { + await this.models[`${operation}_response`].destroy({ + where: { + id: { [Sequelize.Op.in]: ids }, + }, + }); + } } export default OperationResponseRepository; diff --git a/src/modules/repository/repository-module-manager.js b/src/modules/repository/repository-module-manager.js index 688cebe9d..b26729264 100644 --- a/src/modules/repository/repository-module-manager.js +++ b/src/modules/repository/repository-module-manager.js @@ -56,8 +56,32 @@ class RepositoryModuleManager extends BaseModuleManager { return this.getRepository('command').getCommandWithId(id); } - async removeFinalizedCommands(finalizedStatuses) { - return this.getRepository('command').removeFinalizedCommands(finalizedStatuses); + async removeCommands(ids) { + return this.getRepository('command').removeCommands(ids); + } + + async findFinalizedCommands(timestamp, limit) { + return this.getRepository('command').findFinalizedCommands(timestamp, limit); + } + + async findProcessedOperationResponse(timestamp, limit, operation) { + return this.getRepository('operation_response').findProcessedOperationResponse( + timestamp, + limit, + operation, + ); + } + + async removeOperationResponse(ids, operation) { + return this.getRepository('operation_response').removeOperationResponse(ids, operation); + } + + async removeEvents(ids) { + return this.getRepository('blockchain_event').removeEvents(ids); + } + + async findProcessedEvents(timestamp, limit) { + return this.getRepository('blockchain_event').findProcessedEvents(timestamp, limit); } async createOperationIdRecord(handlerData) { diff --git a/src/service/archive-service.js b/src/service/archive-service.js new file mode 100644 index 000000000..1fda755ae --- /dev/null +++ b/src/service/archive-service.js @@ -0,0 +1,19 @@ +class ArchiveService { + constructor(ctx) { + this.logger = ctx.logger; + this.fileService = ctx.fileService; + } + + async archiveData(archiveFolderName, archiveFileName, data) { + const archiveFolderPath = this.fileService.getArchiveFolderPath(archiveFolderName); + this.logger.debug( + `Archiving data on path: ${archiveFolderPath} with archive name: ${archiveFileName}`, + ); + await this.fileService.writeContentsToFile( + archiveFolderPath, + archiveFileName, + JSON.stringify(data), + ); + } +} +export default ArchiveService; diff --git a/src/service/file-service.js b/src/service/file-service.js index b428bf18f..847bfada0 100644 --- a/src/service/file-service.js +++ b/src/service/file-service.js @@ -4,6 +4,8 @@ import appRootPath from 'app-root-path'; const MIGRATION_FOLDER_NAME = 'migrations'; +const ARCHIVE_FOLDER_NAME = 'archive'; + class FileService { constructor(ctx) { this.config = ctx.config; @@ -120,6 +122,10 @@ class FileService { this.getPendingStorageFileName(blockchain, contract, tokenId), ); } + + getArchiveFolderPath(subFolder) { + return path.join(this.getDataFolderPath(), ARCHIVE_FOLDER_NAME, subFolder); + } } export default FileService; From 5df445faa5efaca4b04bcad16f1131a7c4d3371a Mon Sep 17 00:00:00 2001 From: zeroxbt <89495162+zeroxbt@users.noreply.github.com> Date: Mon, 19 Jun 2023 10:06:48 +0200 Subject: [PATCH 11/33] delete operation id files in batches (#2582) --- .../cleaners/operation-id-cleaner-command.js | 2 ++ src/constants/constants.js | 2 ++ src/service/operation-id-service.js | 18 ++++++++++++++---- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/src/commands/cleaners/operation-id-cleaner-command.js b/src/commands/cleaners/operation-id-cleaner-command.js index 7f43a4382..d1816e7d4 100644 --- a/src/commands/cleaners/operation-id-cleaner-command.js +++ b/src/commands/cleaners/operation-id-cleaner-command.js @@ -1,6 +1,7 @@ import Command from '../command.js'; import { BYTES_IN_KILOBYTE, + OPERATION_ID_FILES_FOR_REMOVAL_MAX_NUMBER, OPERATION_ID_COMMAND_CLEANUP_TIME_MILLS, OPERATION_ID_STATUS, } from '../../constants/constants.js'; @@ -39,6 +40,7 @@ class OperationIdCleanerCommand extends Command { } removed = await this.operationIdService.removeExpiredOperationIdFileCache( OPERATION_ID_COMMAND_CLEANUP_TIME_MILLS, + OPERATION_ID_FILES_FOR_REMOVAL_MAX_NUMBER, ); if (removed) { this.logger.debug(`Successfully removed ${removed} expired cached operation files`); diff --git a/src/constants/constants.js b/src/constants/constants.js index 0999842ab..f5dd337c3 100644 --- a/src/constants/constants.js +++ b/src/constants/constants.js @@ -371,6 +371,8 @@ export const COMMAND_STATUS = { REPEATING: 'REPEATING', }; +export const OPERATION_ID_FILES_FOR_REMOVAL_MAX_NUMBER = 100; + export const REPOSITORY_ROWS_FOR_REMOVAL_MAX_NUMBER = 1000; export const ARCHIVE_COMMANDS_FOLDER = 'commands'; diff --git a/src/service/operation-id-service.js b/src/service/operation-id-service.js index 40ead619f..e882317e9 100644 --- a/src/service/operation-id-service.js +++ b/src/service/operation-id-service.js @@ -144,16 +144,17 @@ class OperationIdService { return deleted; } - async removeExpiredOperationIdFileCache(expiredTimeout) { + async removeExpiredOperationIdFileCache(expiredTimeout, batchSize) { const cacheFolderPath = this.fileService.getOperationIdCachePath(); const cacheFolderExists = await this.fileService.fileExists(cacheFolderPath); if (!cacheFolderExists) { return; } const fileList = await this.fileService.readDirectory(cacheFolderPath); + + const now = new Date(); const deleteFile = async (fileName) => { const filePath = path.join(cacheFolderPath, fileName); - const now = new Date(); const createdDate = (await this.fileService.stat(filePath)).mtime; if (createdDate.getTime() + expiredTimeout < now.getTime()) { await this.fileService.removeFile(filePath); @@ -161,8 +162,17 @@ class OperationIdService { } return false; }; - const deleted = await Promise.all(fileList.map((fileName) => deleteFile(fileName))); - return deleted.filter((x) => x).length; + let totalDeleted = 0; + for (let i = 0; i < fileList.length; i += batchSize) { + const batch = fileList.slice(i, i + batchSize); + // eslint-disable-next-line no-await-in-loop + const deletionResults = await Promise.allSettled(batch.map(deleteFile)); + totalDeleted += deletionResults.filter( + (result) => result.status === 'fulfilled' && result.value, + ).length; + } + + return totalDeleted; } } From 95155c95a7f642483c15f0fd72870374e2bc5890 Mon Sep 17 00:00:00 2001 From: zeroxbt <89495162+zeroxbt@users.noreply.github.com> Date: Mon, 19 Jun 2023 10:56:37 +0200 Subject: [PATCH 12/33] add operation cleaner commands (#2581) * add operation cleaner commands * add permanent commands, minor bug fix --- .../blockchain-event-cleaner-command.js | 7 --- src/commands/cleaners/cleaner-command.js | 1 - .../cleaners/commands-cleaner-command.js | 7 --- src/commands/cleaners/get-cleaner-command.js | 44 +++++++++++++++++ .../cleaners/get-response-cleaner-command.js | 7 --- .../cleaners/publish-cleaner-command.js | 44 +++++++++++++++++ .../publish-response-cleaner-command.js | 7 --- .../cleaners/update-cleaner-command.js | 44 +++++++++++++++++ .../update-response-cleaner-command.js | 7 --- src/constants/constants.js | 21 ++++++++ .../repositories/operation-repository.js | 21 ++++++++ .../repository/repository-module-manager.js | 48 +++++++++++-------- 12 files changed, 202 insertions(+), 56 deletions(-) create mode 100644 src/commands/cleaners/get-cleaner-command.js create mode 100644 src/commands/cleaners/publish-cleaner-command.js create mode 100644 src/commands/cleaners/update-cleaner-command.js diff --git a/src/commands/cleaners/blockchain-event-cleaner-command.js b/src/commands/cleaners/blockchain-event-cleaner-command.js index 9eda2c1f0..ea0017793 100644 --- a/src/commands/cleaners/blockchain-event-cleaner-command.js +++ b/src/commands/cleaners/blockchain-event-cleaner-command.js @@ -7,13 +7,6 @@ import { import CleanerCommand from './cleaner-command.js'; class BlockchainEventCleanerCommand extends CleanerCommand { - constructor(ctx) { - super(ctx); - this.logger = ctx.logger; - this.repositoryModuleManager = ctx.repositoryModuleManager; - this.archiveService = ctx.archiveService; - } - async findRowsForRemoval(nowTimestamp) { return this.repositoryModuleManager.findProcessedEvents( nowTimestamp - PROCESSED_BLOCKCHAIN_EVENTS_CLEANUP_TIME_DELAY, diff --git a/src/commands/cleaners/cleaner-command.js b/src/commands/cleaners/cleaner-command.js index c27cd00d0..38dc0685f 100644 --- a/src/commands/cleaners/cleaner-command.js +++ b/src/commands/cleaners/cleaner-command.js @@ -4,7 +4,6 @@ import Command from '../command.js'; class CleanerCommand extends Command { constructor(ctx) { super(ctx); - this.logger = ctx.logger; this.repositoryModuleManager = ctx.repositoryModuleManager; this.archiveService = ctx.archiveService; } diff --git a/src/commands/cleaners/commands-cleaner-command.js b/src/commands/cleaners/commands-cleaner-command.js index 94aa70683..746d18233 100644 --- a/src/commands/cleaners/commands-cleaner-command.js +++ b/src/commands/cleaners/commands-cleaner-command.js @@ -6,13 +6,6 @@ import { import CleanerCommand from './cleaner-command.js'; class CommandsCleanerCommand extends CleanerCommand { - constructor(ctx) { - super(ctx); - this.logger = ctx.logger; - this.repositoryModuleManager = ctx.repositoryModuleManager; - this.archiveService = ctx.archiveService; - } - async findRowsForRemoval(nowTimestamp) { return this.repositoryModuleManager.findFinalizedCommands( nowTimestamp, diff --git a/src/commands/cleaners/get-cleaner-command.js b/src/commands/cleaners/get-cleaner-command.js new file mode 100644 index 000000000..399162387 --- /dev/null +++ b/src/commands/cleaners/get-cleaner-command.js @@ -0,0 +1,44 @@ +import CleanerCommand from './cleaner-command.js'; +import { + REPOSITORY_ROWS_FOR_REMOVAL_MAX_NUMBER, + OPERATIONS, + GET_CLEANUP_TIME_DELAY, + GET_CLEANUP_TIME_MILLS, + ARCHIVE_GET_FOLDER, +} from '../../constants/constants.js'; + +class GetCleanerCommand extends CleanerCommand { + async findRowsForRemoval(nowTimestamp) { + return this.repositoryModuleManager.findProcessedOperations( + OPERATIONS.GET, + nowTimestamp - GET_CLEANUP_TIME_DELAY, + REPOSITORY_ROWS_FOR_REMOVAL_MAX_NUMBER, + ); + } + + getArchiveFolderName() { + return ARCHIVE_GET_FOLDER; + } + + async deleteRows(ids) { + return this.repositoryModuleManager.removeOperationRecords(OPERATIONS.GET, ids); + } + + /** + * Builds default command + * @param map + * @returns {{add, data: *, delay: *, deadline: *}} + */ + default(map) { + const command = { + name: 'getCleanerCommand', + data: {}, + period: GET_CLEANUP_TIME_MILLS, + transactional: false, + }; + Object.assign(command, map); + return command; + } +} + +export default GetCleanerCommand; diff --git a/src/commands/cleaners/get-response-cleaner-command.js b/src/commands/cleaners/get-response-cleaner-command.js index ae035d634..d79e90ba0 100644 --- a/src/commands/cleaners/get-response-cleaner-command.js +++ b/src/commands/cleaners/get-response-cleaner-command.js @@ -8,13 +8,6 @@ import { } from '../../constants/constants.js'; class GetResponseCleanerCommand extends CleanerCommand { - constructor(ctx) { - super(ctx); - this.logger = ctx.logger; - this.repositoryModuleManager = ctx.repositoryModuleManager; - this.archiveService = ctx.archiveService; - } - async findRowsForRemoval(nowTimestamp) { return this.repositoryModuleManager.findProcessedOperationResponse( nowTimestamp - GET_RESPONSE_CLEANUP_TIME_DELAY, diff --git a/src/commands/cleaners/publish-cleaner-command.js b/src/commands/cleaners/publish-cleaner-command.js new file mode 100644 index 000000000..cfa93ea85 --- /dev/null +++ b/src/commands/cleaners/publish-cleaner-command.js @@ -0,0 +1,44 @@ +import CleanerCommand from './cleaner-command.js'; +import { + REPOSITORY_ROWS_FOR_REMOVAL_MAX_NUMBER, + OPERATIONS, + PUBLISH_CLEANUP_TIME_DELAY, + PUBLISH_CLEANUP_TIME_MILLS, + ARCHIVE_PUBLISH_FOLDER, +} from '../../constants/constants.js'; + +class PublishCleanerCommand extends CleanerCommand { + async findRowsForRemoval(nowTimestamp) { + return this.repositoryModuleManager.findProcessedOperations( + OPERATIONS.PUBLISH, + nowTimestamp - PUBLISH_CLEANUP_TIME_DELAY, + REPOSITORY_ROWS_FOR_REMOVAL_MAX_NUMBER, + ); + } + + getArchiveFolderName() { + return ARCHIVE_PUBLISH_FOLDER; + } + + async deleteRows(ids) { + return this.repositoryModuleManager.removeOperationRecords(OPERATIONS.PUBLISH, ids); + } + + /** + * Builds default command + * @param map + * @returns {{add, data: *, delay: *, deadline: *}} + */ + default(map) { + const command = { + name: 'publishCleanerCommand', + data: {}, + period: PUBLISH_CLEANUP_TIME_MILLS, + transactional: false, + }; + Object.assign(command, map); + return command; + } +} + +export default PublishCleanerCommand; diff --git a/src/commands/cleaners/publish-response-cleaner-command.js b/src/commands/cleaners/publish-response-cleaner-command.js index 1a3c981be..85ff500af 100644 --- a/src/commands/cleaners/publish-response-cleaner-command.js +++ b/src/commands/cleaners/publish-response-cleaner-command.js @@ -8,13 +8,6 @@ import { } from '../../constants/constants.js'; class PublishResponseCleanerCommand extends CleanerCommand { - constructor(ctx) { - super(ctx); - this.logger = ctx.logger; - this.repositoryModuleManager = ctx.repositoryModuleManager; - this.archiveService = ctx.archiveService; - } - async findRowsForRemoval(nowTimestamp) { return this.repositoryModuleManager.findProcessedOperationResponse( nowTimestamp - PUBLISH_RESPONSE_CLEANUP_TIME_DELAY, diff --git a/src/commands/cleaners/update-cleaner-command.js b/src/commands/cleaners/update-cleaner-command.js new file mode 100644 index 000000000..9f57fae2e --- /dev/null +++ b/src/commands/cleaners/update-cleaner-command.js @@ -0,0 +1,44 @@ +import CleanerCommand from './cleaner-command.js'; +import { + REPOSITORY_ROWS_FOR_REMOVAL_MAX_NUMBER, + OPERATIONS, + UPDATE_CLEANUP_TIME_DELAY, + UPDATE_CLEANUP_TIME_MILLS, + ARCHIVE_UPDATE_FOLDER, +} from '../../constants/constants.js'; + +class UpdateCleanerCommand extends CleanerCommand { + async findRowsForRemoval(nowTimestamp) { + return this.repositoryModuleManager.findProcessedOperations( + OPERATIONS.UPDATE, + nowTimestamp - UPDATE_CLEANUP_TIME_DELAY, + REPOSITORY_ROWS_FOR_REMOVAL_MAX_NUMBER, + ); + } + + getArchiveFolderName() { + return ARCHIVE_UPDATE_FOLDER; + } + + async deleteRows(ids) { + return this.repositoryModuleManager.removeOperationRecords(OPERATIONS.UPDATE, ids); + } + + /** + * Builds default command + * @param map + * @returns {{add, data: *, delay: *, deadline: *}} + */ + default(map) { + const command = { + name: 'publishCleanerCommand', + data: {}, + period: UPDATE_CLEANUP_TIME_MILLS, + transactional: false, + }; + Object.assign(command, map); + return command; + } +} + +export default UpdateCleanerCommand; diff --git a/src/commands/cleaners/update-response-cleaner-command.js b/src/commands/cleaners/update-response-cleaner-command.js index e0f54d9c9..d5066e726 100644 --- a/src/commands/cleaners/update-response-cleaner-command.js +++ b/src/commands/cleaners/update-response-cleaner-command.js @@ -8,13 +8,6 @@ import { } from '../../constants/constants.js'; class UpdateResponseCleanerCommand extends CleanerCommand { - constructor(ctx) { - super(ctx); - this.logger = ctx.logger; - this.repositoryModuleManager = ctx.repositoryModuleManager; - this.archiveService = ctx.archiveService; - } - async findRowsForRemoval(nowTimestamp) { return this.repositoryModuleManager.findProcessedOperationResponse( nowTimestamp - UPDATE_RESPONSE_CLEANUP_TIME_DELAY, diff --git a/src/constants/constants.js b/src/constants/constants.js index f5dd337c3..ba0523252 100644 --- a/src/constants/constants.js +++ b/src/constants/constants.js @@ -131,8 +131,11 @@ export const PERMANENT_COMMANDS = [ 'dialPeersCommand', 'epochCheckCommand', 'blockchainEventCleanerCommand', + 'getCleanerCommand', 'getResponseCleanerCommand', + 'publishCleanerCommand', 'publishResponseCleanerCommand', + 'updateCleanerCommand', 'updateResponseCleanerCommand', ]; @@ -342,14 +345,26 @@ export const OPERATION_ID_COMMAND_CLEANUP_TIME_MILLS = 24 * 60 * 60 * 1000; */ export const FINALIZED_COMMAND_CLEANUP_TIME_MILLS = 24 * 60 * 60 * 1000; +export const GET_CLEANUP_TIME_MILLS = 24 * 60 * 60 * 1000; + +export const GET_CLEANUP_TIME_DELAY = 24 * 60 * 60 * 1000; + export const GET_RESPONSE_CLEANUP_TIME_MILLS = 24 * 60 * 60 * 1000; export const GET_RESPONSE_CLEANUP_TIME_DELAY = 24 * 60 * 60 * 1000; +export const PUBLISH_CLEANUP_TIME_MILLS = 24 * 60 * 60 * 1000; + +export const PUBLISH_CLEANUP_TIME_DELAY = 24 * 60 * 60 * 1000; + export const PUBLISH_RESPONSE_CLEANUP_TIME_MILLS = 24 * 60 * 60 * 1000; export const PUBLISH_RESPONSE_CLEANUP_TIME_DELAY = 24 * 60 * 60 * 1000; +export const UPDATE_CLEANUP_TIME_MILLS = 24 * 60 * 60 * 1000; + +export const UPDATE_CLEANUP_TIME_DELAY = 24 * 60 * 60 * 1000; + export const UPDATE_RESPONSE_CLEANUP_TIME_MILLS = 24 * 60 * 60 * 1000; export const UPDATE_RESPONSE_CLEANUP_TIME_DELAY = 24 * 60 * 60 * 1000; @@ -379,10 +394,16 @@ export const ARCHIVE_COMMANDS_FOLDER = 'commands'; export const ARCHIVE_BLOCKCHAIN_EVENTS_FOLDER = 'blockchain_events'; +export const ARCHIVE_GET_FOLDER = 'get'; + export const ARCHIVE_GET_RESPONSES_FOLDER = 'get_responses'; +export const ARCHIVE_PUBLISH_FOLDER = 'publish'; + export const ARCHIVE_PUBLISH_RESPONSES_FOLDER = 'publish_responses'; +export const ARCHIVE_UPDATE_FOLDER = 'update'; + export const ARCHIVE_UPDATE_RESPONSES_FOLDER = 'update_responses'; /** diff --git a/src/modules/repository/implementation/sequelize/repositories/operation-repository.js b/src/modules/repository/implementation/sequelize/repositories/operation-repository.js index 63efad192..e0634aacc 100644 --- a/src/modules/repository/implementation/sequelize/repositories/operation-repository.js +++ b/src/modules/repository/implementation/sequelize/repositories/operation-repository.js @@ -1,3 +1,5 @@ +import { Sequelize } from 'sequelize'; + class OperationRepository { constructor(models) { this.sequelize = models.sequelize; @@ -11,6 +13,25 @@ class OperationRepository { }); } + async removeOperationRecords(operation, ids) { + return this.models[operation].destroy({ + where: { + id: { [Sequelize.Op.in]: ids }, + }, + }); + } + + async findProcessedOperations(operation, timestamp, limit) { + return this.models[`${operation}`].findAll({ + where: { + createdAt: { [Sequelize.Op.lte]: timestamp }, + }, + order: [['createdAt', 'asc']], + raw: true, + limit, + }); + } + async getOperationStatus(operation, operationId) { return this.models[operation].findOne({ attributes: ['status'], diff --git a/src/modules/repository/repository-module-manager.js b/src/modules/repository/repository-module-manager.js index b26729264..fd1cc4617 100644 --- a/src/modules/repository/repository-module-manager.js +++ b/src/modules/repository/repository-module-manager.js @@ -64,26 +64,6 @@ class RepositoryModuleManager extends BaseModuleManager { return this.getRepository('command').findFinalizedCommands(timestamp, limit); } - async findProcessedOperationResponse(timestamp, limit, operation) { - return this.getRepository('operation_response').findProcessedOperationResponse( - timestamp, - limit, - operation, - ); - } - - async removeOperationResponse(ids, operation) { - return this.getRepository('operation_response').removeOperationResponse(ids, operation); - } - - async removeEvents(ids) { - return this.getRepository('blockchain_event').removeEvents(ids); - } - - async findProcessedEvents(timestamp, limit) { - return this.getRepository('blockchain_event').findProcessedEvents(timestamp, limit); - } - async createOperationIdRecord(handlerData) { return this.getRepository('operation_id').createOperationIdRecord(handlerData); } @@ -111,6 +91,14 @@ class RepositoryModuleManager extends BaseModuleManager { ); } + async removeOperationRecords(operation, ids) { + return this.getRepository('operation').removeOperationRecords(operation, ids); + } + + async findProcessedOperations(operation, timestamp, limit) { + return this.getRepository('operation').findProcessedOperations(operation, timestamp, limit); + } + async getOperationStatus(operation, operationId) { return this.getRepository('operation').getOperationStatus(operation, operationId); } @@ -140,6 +128,18 @@ class RepositoryModuleManager extends BaseModuleManager { ); } + async findProcessedOperationResponse(timestamp, limit, operation) { + return this.getRepository('operation_response').findProcessedOperationResponse( + timestamp, + limit, + operation, + ); + } + + async removeOperationResponse(ids, operation) { + return this.getRepository('operation_response').removeOperationResponse(ids, operation); + } + // Sharding Table async createManyPeerRecords(peers) { return this.getRepository('shard').createManyPeerRecords(peers); @@ -262,6 +262,14 @@ class RepositoryModuleManager extends BaseModuleManager { return this.getRepository('blockchain_event').removeBlockchainEvents(contract); } + async removeEvents(ids) { + return this.getRepository('blockchain_event').removeEvents(ids); + } + + async findProcessedEvents(timestamp, limit) { + return this.getRepository('blockchain_event').findProcessedEvents(timestamp, limit); + } + async removeLastCheckedBlockForContract(contract) { return this.getRepository('blockchain').removeLastCheckedBlockForContract(contract); } From e8632d8879d9f7ee85603fb52c7234546f3052e3 Mon Sep 17 00:00:00 2001 From: zeroxbt <89495162+zeroxbt@users.noreply.github.com> Date: Mon, 19 Jun 2023 14:50:43 +0200 Subject: [PATCH 13/33] increase network api rate limit and spam detection (#2585) --- src/constants/constants.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/constants/constants.js b/src/constants/constants.js index ba0523252..4f73f7bae 100644 --- a/src/constants/constants.js +++ b/src/constants/constants.js @@ -95,12 +95,12 @@ export const MIN_NODE_VERSION = 16; export const NETWORK_API_RATE_LIMIT = { TIME_WINDOW_MILLS: 1 * 60 * 1000, - MAX_NUMBER: 20, + MAX_NUMBER: 100, }; export const NETWORK_API_SPAM_DETECTION = { TIME_WINDOW_MILLS: 1 * 60 * 1000, - MAX_NUMBER: 40, + MAX_NUMBER: 150, }; export const NETWORK_API_BLACK_LIST_TIME_WINDOW_MINUTES = 60; From c0a08868ec68a53afcab05ad588e71869e8b750c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 Jun 2023 15:11:21 +0200 Subject: [PATCH 14/33] Bump dottie from 2.0.3 to 2.0.4 (#2577) Bumps [dottie](https://github.com/mickhansen/dottie.js) from 2.0.3 to 2.0.4. - [Release notes](https://github.com/mickhansen/dottie.js/releases) - [Commits](https://github.com/mickhansen/dottie.js/compare/v2.0.3...v2.0.4) --- updated-dependencies: - dependency-name: dottie dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: zeroxbt <89495162+zeroxbt@users.noreply.github.com> --- package-lock.json | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/package-lock.json b/package-lock.json index 611b263c6..ca1e3258b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8283,9 +8283,9 @@ } }, "node_modules/dottie": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/dottie/-/dottie-2.0.3.tgz", - "integrity": "sha512-4liA0PuRkZWQFQjwBypdxPfZaRWiv5tkhMXY2hzsa2pNf5s7U3m9cwUchfNKe8wZQxdGPQQzO6Rm2uGe0rvohQ==" + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/dottie/-/dottie-2.0.4.tgz", + "integrity": "sha512-iz64WUOmp/ECQhWMJjTWFzJN/wQ7RJ5v/a6A2OiCwjaGCpNo66WGIjlSf+IULO9DQd0b4cFawLOTbiKSrpKodw==" }, "node_modules/duplexer2": { "version": "0.1.4", @@ -26004,9 +26004,9 @@ "integrity": "sha512-7GO6HghkA5fYG9TYnNxi14/7K9f5occMlp3zXAuSxn7CKCxt9xbNWG7yF8hTCSUchlfWSe3uLmlPfigevRItzQ==" }, "dottie": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/dottie/-/dottie-2.0.3.tgz", - "integrity": "sha512-4liA0PuRkZWQFQjwBypdxPfZaRWiv5tkhMXY2hzsa2pNf5s7U3m9cwUchfNKe8wZQxdGPQQzO6Rm2uGe0rvohQ==" + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/dottie/-/dottie-2.0.4.tgz", + "integrity": "sha512-iz64WUOmp/ECQhWMJjTWFzJN/wQ7RJ5v/a6A2OiCwjaGCpNo66WGIjlSf+IULO9DQd0b4cFawLOTbiKSrpKodw==" }, "duplexer2": { "version": "0.1.4", From 87284fa877572a69dca53d9869dc9e1d28afde7c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 Jun 2023 15:25:10 +0200 Subject: [PATCH 15/33] Bump @openzeppelin/contracts from 4.8.3 to 4.9.1 (#2574) Bumps [@openzeppelin/contracts](https://github.com/OpenZeppelin/openzeppelin-contracts) from 4.8.3 to 4.9.1. - [Release notes](https://github.com/OpenZeppelin/openzeppelin-contracts/releases) - [Changelog](https://github.com/OpenZeppelin/openzeppelin-contracts/blob/v4.9.1/CHANGELOG.md) - [Commits](https://github.com/OpenZeppelin/openzeppelin-contracts/compare/v4.8.3...v4.9.1) --- updated-dependencies: - dependency-name: "@openzeppelin/contracts" dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: zeroxbt <89495162+zeroxbt@users.noreply.github.com> --- package-lock.json | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/package-lock.json b/package-lock.json index ca1e3258b..4873b3883 100644 --- a/package-lock.json +++ b/package-lock.json @@ -4304,9 +4304,9 @@ } }, "node_modules/@openzeppelin/contracts": { - "version": "4.8.3", - "resolved": "https://registry.npmjs.org/@openzeppelin/contracts/-/contracts-4.8.3.tgz", - "integrity": "sha512-bQHV8R9Me8IaJoJ2vPG4rXcL7seB7YVuskr4f+f5RyOStSZetwzkWtoqDMl5erkBJy0lDRUnIR2WIkPiC0GJlg==" + "version": "4.9.1", + "resolved": "https://registry.npmjs.org/@openzeppelin/contracts/-/contracts-4.9.1.tgz", + "integrity": "sha512-aLDTLu/If1qYIFW5g4ZibuQaUsFGWQPBq1mZKp/txaebUnGHDmmiBhRLY1tDNedN0m+fJtKZ1zAODS9Yk+V6uA==" }, "node_modules/@polkadot/api": { "version": "9.14.2", @@ -22809,9 +22809,9 @@ "optional": true }, "@openzeppelin/contracts": { - "version": "4.8.3", - "resolved": "https://registry.npmjs.org/@openzeppelin/contracts/-/contracts-4.8.3.tgz", - "integrity": "sha512-bQHV8R9Me8IaJoJ2vPG4rXcL7seB7YVuskr4f+f5RyOStSZetwzkWtoqDMl5erkBJy0lDRUnIR2WIkPiC0GJlg==" + "version": "4.9.1", + "resolved": "https://registry.npmjs.org/@openzeppelin/contracts/-/contracts-4.9.1.tgz", + "integrity": "sha512-aLDTLu/If1qYIFW5g4ZibuQaUsFGWQPBq1mZKp/txaebUnGHDmmiBhRLY1tDNedN0m+fJtKZ1zAODS9Yk+V6uA==" }, "@polkadot/api": { "version": "9.14.2", From 445f4bee2c492d181cbc5272ea9f047af4bce510 Mon Sep 17 00:00:00 2001 From: Uladzislau Hubar <71610423+u-hubar@users.noreply.github.com> Date: Mon, 19 Jun 2023 16:41:35 +0100 Subject: [PATCH 16/33] Get historical (#2573) * Reworked fileService to work with path patterns, changed naming of the files for pending storage (added stateId to the name), added possibility to get historical states and use hashes as state argument * Fixed minor bugs related to get operation, changed glob to use async/await, reworked BDD tests, added scenarios for get operation to BDD tests * Reworked github actions * Small fixes in github setup action * Fixed run in the github action setup * Added JWT_SECRET to env for checks github workflow, make blazegraph run in the background * Increased timeout for bootstrap node start in BDD tests to 50 seconds * Decreased timeout for bootstrap node loading to 30 sec, added BDD test logs as artifacts in github actions * Make artifacts uploading work even if BDD tests are failed in github actions * Removed password for MySQL in github actions * Reverted MySQL pssword in the github actions and added password to the env * Reverted MySQL host in github actions setup * Changed MySQL host to 127.0.0.1 for DBB tests after hook cleanup * Added catch block for BDD tests after hook * Added contracts cache cleaning with after hook in BDD tests * Removed contracts cache cleaning for local blockchain * Temporary fix for BDD tests * Reverted back nodes number in publish error BDD tests * fix get assertion id command * fix catch error in fileExists * Bumped version to 6.0.11 * fix stateIsPending call * Small fixes after reviews * update pending storage to support get pending state by state id * Reworked File Service, removed unnecessary glob dependency * Updated after hook * Added error handling for readFile in fileService * Updated after hooks * Added logs * Fixes after review * Added missing awaits * Updated the way we are creating profiles in tests * Reverted changes for profile creation * Added removal of data folders after each test * Updated pending storage service * Removed log from file service remove folder * fix migration * Added state hash validation for GET operation, additional BDD tests for GET * Changed condition for state hash validation in GET operation * Fixed datasets for BDD tests --------- Co-authored-by: zeroxbt Co-authored-by: Djordje Kovacevic Co-authored-by: zeroxbt <89495162+zeroxbt@users.noreply.github.com> --- .github/actions/setup/action.yml | 41 +++ .github/workflows/CHECK-lint.yml | 32 --- .github/workflows/TEST-bdd.yml | 64 ----- .github/workflows/TEST-unit.yml | 49 ---- .github/workflows/checks.yml | 77 +++++ .github/workflows/release-drafter-config.yml | 5 +- .github/workflows/update-cache.yml | 22 ++ ot-node.js | 19 ++ package-lock.json | 272 +++++++++++++++--- package.json | 2 +- .../local-store/local-store-command.js | 1 + .../v1.0.0/v1-0-0-handle-get-init-command.js | 19 +- .../v1-0-0-handle-get-request-command.js | 30 +- .../get/sender/get-assertion-id-command.js | 94 ++++++ .../sender/get-latest-assertion-id-command.js | 94 ------ .../protocols/get/sender/local-get-command.js | 42 +-- .../sender/v1.0.0/v1-0-0-get-init-command.js | 12 +- .../v1.0.0/v1-0-0-get-request-command.js | 6 +- .../v1-0-0-handle-update-request-command.js | 5 +- src/constants/constants.js | 4 +- .../http-api/get-http-api-controller.js | 24 +- .../http-api/request-schema/get-schema.js | 10 +- src/controllers/rpc/publish-rpc-controller.js | 1 + src/controllers/rpc/update-rpc-controller.js | 6 +- src/migration/base-migration.js | 2 +- .../blockchain-identity-migration.js | 2 +- src/migration/pending-storage-migration.js | 38 +++ .../service-agreements-metadata-migration.js | 4 +- .../triple-store-metadata-migration.js | 8 +- ...iple-store-user-configuration-migration.js | 7 +- .../blockchain/blockchain-module-manager.js | 7 + .../blockchain/implementation/web3-service.js | 12 +- src/service/file-service.js | 109 ++++--- src/service/operation-id-service.js | 6 +- src/service/pending-storage-service.js | 76 ++--- src/service/validation-service.js | 19 ++ test/bdd/features/get-errors.feature | 51 +++- test/bdd/features/get.feature | 76 +++++ test/bdd/features/publish-errors.feature | 26 +- test/bdd/features/publish.feature | 11 +- test/bdd/features/update-errors.feature | 10 +- test/bdd/features/update.feature | 21 +- test/bdd/steps/api/datasets/assertions.json | 15 + test/bdd/steps/api/datasets/requests.json | 44 ++- test/bdd/steps/api/get.mjs | 105 +++++++ test/bdd/steps/api/info.mjs | 4 +- test/bdd/steps/api/publish.mjs | 55 ++-- test/bdd/steps/api/resolve.mjs | 77 ++--- test/bdd/steps/api/update.mjs | 44 +-- test/bdd/steps/common.mjs | 58 ++-- test/bdd/steps/hooks.mjs | 19 +- test/bdd/steps/lib/local-blockchain.mjs | 21 +- test/bdd/steps/lib/state.mjs | 13 +- test/utilities/dkg-client-helper.mjs | 43 +-- test/utilities/http-api-helper.mjs | 73 ++--- test/utilities/steps-utils.mjs | 4 - 56 files changed, 1295 insertions(+), 696 deletions(-) create mode 100644 .github/actions/setup/action.yml delete mode 100644 .github/workflows/CHECK-lint.yml delete mode 100644 .github/workflows/TEST-bdd.yml delete mode 100644 .github/workflows/TEST-unit.yml create mode 100644 .github/workflows/checks.yml create mode 100644 .github/workflows/update-cache.yml create mode 100644 src/commands/protocols/get/sender/get-assertion-id-command.js delete mode 100644 src/commands/protocols/get/sender/get-latest-assertion-id-command.js create mode 100644 src/migration/pending-storage-migration.js create mode 100644 test/bdd/features/get.feature create mode 100644 test/bdd/steps/api/get.mjs diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml new file mode 100644 index 000000000..9acedddf3 --- /dev/null +++ b/.github/actions/setup/action.yml @@ -0,0 +1,41 @@ +name: setup + +runs: + using: composite + steps: + - name: Setup NodeJS + id: nodejs + uses: actions/setup-node@v3 + with: + node-version: 16.x + cache: npm + + - name: Cache node modules + id: cache-node-modules + uses: actions/cache@v3 + with: + path: '**/node_modules' + key: ${{ runner.os }}-node-modules-${{ hashFiles('**/package-lock.json') }} + restore-keys: | + ${{ runner.os }}-node-modules-${{ hashFiles('**/package-lock.json') }} + ${{ runner.os }}-node-modules- + + - name: Cache Blazegraph + id: cache-blazegraph + uses: actions/cache@v3 + with: + path: blazegraph.jar + key: ${{ runner.os }}-blazegraph-${{ hashFiles('blazegraph.jar') }} + restore-keys: ${{ runner.os }}-blazegraph- + + - if: steps.cache-node-modules.outputs.cache-hit != 'true' + name: Install dependencies & compile contracts + shell: bash + run: | + npm ci + npm explore dkg-evm-module -- npm run compile + + - if: steps.cache-blazegraph.outputs.cache-hit != 'true' + name: Download Blazegraph + shell: bash + run: wget https://github.com/blazegraph/database/releases/latest/download/blazegraph.jar diff --git a/.github/workflows/CHECK-lint.yml b/.github/workflows/CHECK-lint.yml deleted file mode 100644 index a8fca282c..000000000 --- a/.github/workflows/CHECK-lint.yml +++ /dev/null @@ -1,32 +0,0 @@ -name: CHECK-lint - -#todo this test should be execute when opening PR to prerelease/release branches -on: [pull_request] -env: - NODE_ENV: test - ARTIFACTS_DIR: artifacts - CUCUMBER_ARTIFACTS_DIR: artifacts/cucumber -jobs: - check-lint: - #todo think about locking the version - version should be the same as the one in official documentation - runs-on: ubuntu-latest - strategy: - matrix: - node-version: [16.x] - steps: - - uses: actions/checkout@v2 - - name: Use Node.js ${{ matrix.node-version }} - uses: actions/setup-node@v2 - with: - node-version: ${{ matrix.node-version }} - - run: npm install - - run: mkdir -p $ARTIFACTS_DIR - - run: sudo chmod -R 777 $ARTIFACTS_DIR - - run: mkdir -p $CUCUMBER_ARTIFACTS_DIR - - run: sudo chmod -R 777 $CUCUMBER_ARTIFACTS_DIR - - run: npm run lint; - - uses: actions/upload-artifact@v2 - if: ${{ always() }} - with: - name: my-artifact - path: /home/runner/work/ot-node/ot-node/artifacts diff --git a/.github/workflows/TEST-bdd.yml b/.github/workflows/TEST-bdd.yml deleted file mode 100644 index 7fb92f8fe..000000000 --- a/.github/workflows/TEST-bdd.yml +++ /dev/null @@ -1,64 +0,0 @@ -name: TEST-bdd - -#todo this test should be execute when opening PR to prerelease/release branches -on: [pull_request] -env: - NODE_ENV: test - ARTIFACTS_DIR: artifacts - CUCUMBER_ARTIFACTS_DIR: artifacts/cucumber - REPOSITORY_PASSWORD: password -jobs: - - test-bdd: - #todo think about locking the version - version should be the same as the one in official documentation - runs-on: ubuntu-latest - services: - mysql: - image: mysql:5.7 - env: - MYSQL_DATABASE: operationaldb - MYSQL_USER: node - MYSQL_PASSWORD: password - MYSQL_ROOT_PASSWORD: password - ports: - - 3306:3306 - options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3 - strategy: - matrix: - node-version: [16.x] - steps: - - uses: actions/checkout@v2 - - name: Use Node.js ${{ matrix.node-version }} - uses: actions/setup-node@v2 - with: - node-version: ${{ matrix.node-version }} - - name: Download Blazegraph - run: wget https://github.com/blazegraph/database/releases/latest/download/blazegraph.jar - - name: Cache Blazegraph - uses: actions/cache@v2 - with: - path: blazegraph.jar - key: ${{ runner.os }}-blazegraph-${{ hashFiles('blazegraph.jar') }} - restore-keys: ${{ runner.os }}-blazegraph- - - run: /usr/bin/java -Djava.awt.headless=true -jar blazegraph.jar & - - name: Cache node_modules - uses: actions/cache@v2 - with: - path: | - ~/.npm - ./node_modules - key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }} - restore-keys: | - ${{ runner.os }}-node- - - run: npm ci - - run: mkdir -p $ARTIFACTS_DIR - - run: sudo chmod -R 777 $ARTIFACTS_DIR - - run: mkdir -p $CUCUMBER_ARTIFACTS_DIR - - run: sudo chmod -R 777 $CUCUMBER_ARTIFACTS_DIR - - run: npm explore dkg-evm-module -- npm run compile; - - run: npm run test:bdd; - - uses: actions/upload-artifact@v2 - if: ${{ always() }} - with: - name: my-artifact - path: /home/runner/work/ot-node/ot-node/artifacts diff --git a/.github/workflows/TEST-unit.yml b/.github/workflows/TEST-unit.yml deleted file mode 100644 index 9b437e88e..000000000 --- a/.github/workflows/TEST-unit.yml +++ /dev/null @@ -1,49 +0,0 @@ -name: TEST-unit - -#todo this test should be execute when opening PR to prerelease/release branches -on: [pull_request] -env: - NODE_ENV: test - ARTIFACTS_DIR: artifacts - CUCUMBER_ARTIFACTS_DIR: artifacts/cucumber - JWT_SECRET: aTx13FzDG+85j9b5s2G7IBEc5SJNJZZLPLe7RF8hu1xKgRKj46YFRx/z7fJi7iF2NnL7SHcxTzq7TySuPKWkdg/AYKEMD2p1I++qPYFHqg8KQeLArGjCYiqtf43i1Fgtya8z9qJXyegogMz/jYori2BJ8v6b4K3GkAw3XxiO7VaaEYktOp8qsRDcN3b+bITMZqztDvZdWp4EnViGjoES7fRFhKm/d/2C8URnQyGm6xgTR3xTfAjy7+milGmoPA0KU0nu+GsZIhOfeVc9Z2nfxOK/1JQykpjeBhNDYTOr31yW/xdvoW0Kq0PZ6JmM+yezLoyQXcYjavZ+X7cXjbREQg== -jobs: - - test-unit: - #todo think about locking the version - version should be the same as the one in official documentation - runs-on: ubuntu-latest - services: -# mysql: -# image: mysql:5.7 -# env: -# MYSQL_DATABASE: operationaldb -# MYSQL_USER: node -# MYSQL_PASSWORD: password -# MYSQL_ROOT_PASSWORD: password -# ports: -# - 3306:3306 -# options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3 - graphdb: - image: khaller/graphdb-free:latest - ports: - - 7200:7200 - strategy: - matrix: - node-version: [16.x] - steps: - - uses: actions/checkout@v2 - - name: Use Node.js ${{ matrix.node-version }} - uses: actions/setup-node@v2 - with: - node-version: ${{ matrix.node-version }} - - run: npm install - - run: mkdir -p $ARTIFACTS_DIR - - run: sudo chmod -R 777 $ARTIFACTS_DIR - - run: mkdir -p $CUCUMBER_ARTIFACTS_DIR - - run: sudo chmod -R 777 $CUCUMBER_ARTIFACTS_DIR - - run: npm run test:unit; - - uses: actions/upload-artifact@v2 - if: ${{ always() }} - with: - name: my-artifact - path: /home/runner/work/ot-node/ot-node/artifacts diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml new file mode 100644 index 000000000..64356fb2f --- /dev/null +++ b/.github/workflows/checks.yml @@ -0,0 +1,77 @@ +name: checks + +on: + pull_request: + branches: + - v6/develop + +env: + REPOSITORY_PASSWORD: password + JWT_SECRET: aTx13FzDG+85j9b5s2G7IBEc5SJNJZZLPLe7RF8hu1xKgRKj46YFRx/z7fJi7iF2NnL7SHcxTzq7TySuPKWkdg/AYKEMD2p1I++qPYFHqg8KQeLArGjCYiqtf43i1Fgtya8z9qJXyegogMz/jYori2BJ8v6b4K3GkAw3XxiO7VaaEYktOp8qsRDcN3b+bITMZqztDvZdWp4EnViGjoES7fRFhKm/d/2C8URnQyGm6xgTR3xTfAjy7+milGmoPA0KU0nu+GsZIhOfeVc9Z2nfxOK/1JQykpjeBhNDYTOr31yW/xdvoW0Kq0PZ6JmM+yezLoyQXcYjavZ+X7cXjbREQg== + +concurrency: + group: checks-${{ github.ref }} + cancel-in-progress: true + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Set up environment + uses: ./.github/actions/setup + + - name: Run linter + run: npm run lint + + unit-tests: + runs-on: ubuntu-latest + services: + graphdb: + image: khaller/graphdb-free:latest + ports: + - 7200:7200 + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Set up environment + uses: ./.github/actions/setup + + - name: Run unit tests + run: npm run test:unit + + bdd-tests: + runs-on: ubuntu-latest + services: + mysql: + image: mysql:5.7 + env: + MYSQL_DATABASE: operationaldb + MYSQL_USER: node + MYSQL_PASSWORD: password + MYSQL_ROOT_PASSWORD: password + ports: + - 3306:3306 + options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3 + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Set up environment + uses: ./.github/actions/setup + + - name: Run Blazegraph + run: /usr/bin/java -Djava.awt.headless=true -jar blazegraph.jar & + + - name: Run BDD tests + run: npm run test:bdd + + - name: Upload log files + if: '!cancelled()' + uses: actions/upload-artifact@v3 + with: + name: bdd-test-logs + path: ./test/bdd/log/ diff --git a/.github/workflows/release-drafter-config.yml b/.github/workflows/release-drafter-config.yml index 705fdc435..e48dfad82 100644 --- a/.github/workflows/release-drafter-config.yml +++ b/.github/workflows/release-drafter-config.yml @@ -1,10 +1,9 @@ -name: Release Drafter +name: release-drafter on: push: - # branches to consider in the event; optional, defaults to all branches: - - develop + - v6/develop jobs: update_release_draft: diff --git a/.github/workflows/update-cache.yml b/.github/workflows/update-cache.yml new file mode 100644 index 000000000..cbf1f24c6 --- /dev/null +++ b/.github/workflows/update-cache.yml @@ -0,0 +1,22 @@ +name: update-cache + +on: + push: + branches: + - v6/develop + +concurrency: + group: update-cache-${{ github.ref }} + cancel-in-progress: true + +jobs: + build-cache: + name: Build Cache + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Set up environment + uses: ./.github/actions/setup diff --git a/ot-node.js b/ot-node.js index 17407c389..9ac39cf52 100644 --- a/ot-node.js +++ b/ot-node.js @@ -17,6 +17,7 @@ import RemoveAgreementStartEndTimeMigration from './src/migration/remove-agreeme import MarkOldBlockchainEventsAsProcessedMigration from './src/migration/mark-old-blockchain-events-as-processed-migration.js'; import TripleStoreMetadataMigration from './src/migration/triple-store-metadata-migration.js'; import RemoveOldEpochCommandsMigration from './src/migration/remove-old-epoch-commands-migration.js'; +import PendingStorageMigration from './src/migration/pending-storage-migration.js'; const require = createRequire(import.meta.url); const pjson = require('./package.json'); @@ -58,6 +59,7 @@ class OTNode { await this.executeTripleStoreMetadataMigration(); await this.executeServiceAgreementsMetadataMigration(); await this.executeRemoveOldEpochCommandsMigration(); + await this.executePendingStorageMigration(); await this.createProfiles(); @@ -421,6 +423,23 @@ class OTNode { } } + async executePendingStorageMigration() { + if ( + process.env.NODE_ENV === NODE_ENVIRONMENTS.DEVELOPMENT || + process.env.NODE_ENV === NODE_ENVIRONMENTS.TEST + ) + return; + + const migration = new PendingStorageMigration( + 'pendingStorageMigration', + this.logger, + this.config, + ); + if (!(await migration.migrationAlreadyExecuted())) { + await migration.migrate(); + } + } + async initializeShardingTableService() { try { const shardingTableService = this.container.resolve('shardingTableService'); diff --git a/package-lock.json b/package-lock.json index 4873b3883..90f9414b5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "origintrail_node", - "version": "6.0.10", + "version": "6.0.11", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "origintrail_node", - "version": "6.0.10", + "version": "6.0.11", "license": "ISC", "dependencies": { "@comunica/query-sparql": "^2.4.3", @@ -2687,6 +2687,26 @@ "regexp-match-indices": "1.0.2" } }, + "node_modules/@cucumber/cucumber/node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "dev": true, + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/@cucumber/gherkin": { "version": "26.0.3", "resolved": "https://registry.npmjs.org/@cucumber/gherkin/-/gherkin-26.0.3.tgz", @@ -9984,6 +10004,25 @@ "rimraf": "^2.6.3" } }, + "node_modules/fs-jetpack/node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/fs-jetpack/node_modules/rimraf": { "version": "2.7.1", "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.7.1.tgz", @@ -10036,6 +10075,25 @@ "node": ">=0.6" } }, + "node_modules/fstream/node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/fstream/node_modules/rimraf": { "version": "2.7.1", "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.7.1.tgz", @@ -10181,25 +10239,6 @@ "assert-plus": "^1.0.0" } }, - "node_modules/glob": { - "version": "7.2.3", - "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", - "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", - "dependencies": { - "fs.realpath": "^1.0.0", - "inflight": "^1.0.4", - "inherits": "2", - "minimatch": "^3.1.1", - "once": "^1.3.0", - "path-is-absolute": "^1.0.0" - }, - "engines": { - "node": "*" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, "node_modules/glob-parent": { "version": "6.0.2", "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", @@ -14522,6 +14561,26 @@ "node": ">=8" } }, + "node_modules/nyc/node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "dev": true, + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/nyc/node_modules/locate-path": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", @@ -16258,6 +16317,25 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/rimraf/node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/ripemd160": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/ripemd160/-/ripemd160-2.0.2.tgz", @@ -16922,6 +17000,25 @@ "rimraf": "^2.2.8" } }, + "node_modules/solc/node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/solc/node_modules/jsonfile": { "version": "2.4.0", "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-2.4.0.tgz", @@ -17652,6 +17749,26 @@ "node": ">=8" } }, + "node_modules/test-exclude/node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "dev": true, + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/text-hex": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/text-hex/-/text-hex-1.0.0.tgz", @@ -21737,6 +21854,22 @@ "xmlbuilder": "^15.1.1", "yaml": "1.10.2", "yup": "^0.32.11" + }, + "dependencies": { + "glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "dev": true, + "requires": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + } + } } }, "@cucumber/cucumber-expressions": { @@ -27390,6 +27523,19 @@ "rimraf": "^2.6.3" }, "dependencies": { + "glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "requires": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + } + }, "rimraf": { "version": "2.7.1", "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.7.1.tgz", @@ -27431,6 +27577,19 @@ "rimraf": "2" }, "dependencies": { + "glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "requires": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + } + }, "rimraf": { "version": "2.7.1", "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.7.1.tgz", @@ -27539,19 +27698,6 @@ "assert-plus": "^1.0.0" } }, - "glob": { - "version": "7.2.3", - "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", - "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", - "requires": { - "fs.realpath": "^1.0.0", - "inflight": "^1.0.4", - "inherits": "2", - "minimatch": "^3.1.1", - "once": "^1.3.0", - "path-is-absolute": "^1.0.0" - } - }, "glob-parent": { "version": "6.0.2", "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", @@ -30919,6 +31065,20 @@ "path-exists": "^4.0.0" } }, + "glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "dev": true, + "requires": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + } + }, "locate-path": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", @@ -32265,6 +32425,21 @@ "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==", "requires": { "glob": "^7.1.3" + }, + "dependencies": { + "glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "requires": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + } + } } }, "ripemd160": { @@ -32767,6 +32942,19 @@ "rimraf": "^2.2.8" } }, + "glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "requires": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + } + }, "jsonfile": { "version": "2.4.0", "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-2.4.0.tgz", @@ -33342,6 +33530,22 @@ "@istanbuljs/schema": "^0.1.2", "glob": "^7.1.4", "minimatch": "^3.0.4" + }, + "dependencies": { + "glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "dev": true, + "requires": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + } + } } }, "text-hex": { diff --git a/package.json b/package.json index c2d531f07..aaeca8c03 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "origintrail_node", - "version": "6.0.10", + "version": "6.0.11", "description": "OTNode V6", "main": "index.js", "type": "module", diff --git a/src/commands/local-store/local-store-command.js b/src/commands/local-store/local-store-command.js index 7a8f5c8db..8719f8d32 100644 --- a/src/commands/local-store/local-store-command.js +++ b/src/commands/local-store/local-store-command.js @@ -80,6 +80,7 @@ class LocalStoreCommand extends Command { blockchain, contract, tokenId, + cachedData.public.assertionId, { ...cachedData, keyword, diff --git a/src/commands/protocols/get/receiver/v1.0.0/v1-0-0-handle-get-init-command.js b/src/commands/protocols/get/receiver/v1.0.0/v1-0-0-handle-get-init-command.js index 48d3b3dec..eb8edd19d 100644 --- a/src/commands/protocols/get/receiver/v1.0.0/v1-0-0-handle-get-init-command.js +++ b/src/commands/protocols/get/receiver/v1.0.0/v1-0-0-handle-get-init-command.js @@ -19,7 +19,7 @@ class HandleGetInitCommand extends HandleProtocolMessageCommand { } async prepareMessage(commandData) { - const { assertionId, operationId, state, blockchain, contract, tokenId } = commandData; + const { operationId, blockchain, contract, tokenId, assertionId, state } = commandData; await this.operationIdService.updateOperationIdStatus( operationId, @@ -32,12 +32,12 @@ class HandleGetInitCommand extends HandleProtocolMessageCommand { let assertionExists; if ( - state === GET_STATES.LATEST && + state !== GET_STATES.FINALIZED && blockchain != null && contract != null && tokenId != null ) { - assertionExists = await this.pendingStorageService.assertionExists( + assertionExists = await this.pendingStorageService.assetHasPendingState( PENDING_STORAGE_REPOSITORIES.PUBLIC, blockchain, contract, @@ -45,9 +45,18 @@ class HandleGetInitCommand extends HandleProtocolMessageCommand { operationId, ); } - if (!assertionExists) { + + for (const repository of [ + TRIPLE_STORE_REPOSITORIES.PUBLIC_CURRENT, + TRIPLE_STORE_REPOSITORIES.PUBLIC_HISTORY, + ]) { + if (assertionExists) { + break; + } + + // eslint-disable-next-line no-await-in-loop assertionExists = await this.tripleStoreService.assertionExists( - TRIPLE_STORE_REPOSITORIES.PUBLIC_CURRENT, + repository, assertionId, ); } diff --git a/src/commands/protocols/get/receiver/v1.0.0/v1-0-0-handle-get-request-command.js b/src/commands/protocols/get/receiver/v1.0.0/v1-0-0-handle-get-request-command.js index 5f63b8011..55adc3989 100644 --- a/src/commands/protocols/get/receiver/v1.0.0/v1-0-0-handle-get-request-command.js +++ b/src/commands/protocols/get/receiver/v1.0.0/v1-0-0-handle-get-request-command.js @@ -20,23 +20,23 @@ class HandleGetRequestCommand extends HandleProtocolMessageCommand { } async prepareMessage(commandData) { - const { assertionId, operationId, state } = commandData; + const { operationId, blockchain, contract, tokenId, assertionId, state } = commandData; await this.operationIdService.updateOperationIdStatus( operationId, OPERATION_ID_STATUS.GET.GET_REMOTE_START, ); if ( - state === GET_STATES.LATEST && - commandData.blockchain != null && - commandData.contract != null && - commandData.tokenId != null + state !== GET_STATES.FINALIZED && + blockchain != null && + contract != null && + tokenId != null ) { const cachedAssertion = await this.pendingStorageService.getCachedAssertion( PENDING_STORAGE_REPOSITORIES.PUBLIC, - commandData.blockchain, - commandData.contract, - commandData.tokenId, + blockchain, + contract, + tokenId, operationId, ); if (cachedAssertion?.public?.assertion?.length) { @@ -47,10 +47,18 @@ class HandleGetRequestCommand extends HandleProtocolMessageCommand { } } - const nquads = await this.tripleStoreService.getAssertion( + let nquads; + for (const repository of [ TRIPLE_STORE_REPOSITORIES.PUBLIC_CURRENT, - assertionId, - ); + TRIPLE_STORE_REPOSITORIES.PUBLIC_HISTORY, + ]) { + // eslint-disable-next-line no-await-in-loop + nquads = await this.tripleStoreService.getAssertion(repository, assertionId); + + if (nquads.length) { + break; + } + } await this.operationIdService.updateOperationIdStatus( operationId, diff --git a/src/commands/protocols/get/sender/get-assertion-id-command.js b/src/commands/protocols/get/sender/get-assertion-id-command.js new file mode 100644 index 000000000..ca7629c45 --- /dev/null +++ b/src/commands/protocols/get/sender/get-assertion-id-command.js @@ -0,0 +1,94 @@ +import Command from '../../../command.js'; +import { ERROR_TYPE, GET_STATES, ZERO_BYTES32 } from '../../../../constants/constants.js'; + +class GetAssertionIdCommand extends Command { + constructor(ctx) { + super(ctx); + this.operationService = ctx.getService; + this.blockchainModuleManager = ctx.blockchainModuleManager; + this.ualService = ctx.ualService; + + this.errorType = ERROR_TYPE.GET.GET_ASSERTION_ID_ERROR; + } + + /** + * Executes command and produces one or more events + * @param command + */ + async execute(command) { + const { operationId, blockchain, contract, tokenId, state } = command.data; + + let assertionId; + if (!Object.values(GET_STATES).includes(state)) { + const pendingState = await this.blockchainModuleManager.getUnfinalizedAssertionId( + blockchain, + tokenId, + ); + + if ( + (pendingState !== ZERO_BYTES32 && pendingState !== state) || + (pendingState === ZERO_BYTES32 && + !( + await this.blockchainModuleManager.getAssertionIds( + blockchain, + contract, + tokenId, + ) + ).includes(state)) + ) { + await this.handleError( + operationId, + `Given state: ${state} doesn't exist on ${blockchain} on contract: ${contract} for Knowledge Asset with tokenId: ${tokenId}`, + this.errorType, + ); + + return Command.empty(); + } + assertionId = state; + } else { + this.logger.debug( + `Searching for latest assertion id on ${blockchain} on contract: ${contract} with tokenId: ${tokenId}`, + ); + + if (state === GET_STATES.LATEST) { + assertionId = await this.blockchainModuleManager.getUnfinalizedAssertionId( + blockchain, + tokenId, + ); + } + if (assertionId == null || parseInt(assertionId, 16) === 0) { + assertionId = await this.blockchainModuleManager.getLatestAssertionId( + blockchain, + contract, + tokenId, + ); + } + } + + return this.continueSequence( + { ...command.data, state: assertionId, assertionId }, + command.sequence, + ); + } + + async handleError(operationId, errorMessage, errorType) { + await this.operationService.markOperationAsFailed(operationId, errorMessage, errorType); + } + + /** + * Builds default getStateIdConditionalCommand + * @param map + * @returns {{add, data: *, delay: *, deadline: *}} + */ + default(map) { + const command = { + name: 'getAssertionIdCommand', + delay: 0, + transactional: false, + }; + Object.assign(command, map); + return command; + } +} + +export default GetAssertionIdCommand; diff --git a/src/commands/protocols/get/sender/get-latest-assertion-id-command.js b/src/commands/protocols/get/sender/get-latest-assertion-id-command.js deleted file mode 100644 index 362e66413..000000000 --- a/src/commands/protocols/get/sender/get-latest-assertion-id-command.js +++ /dev/null @@ -1,94 +0,0 @@ -import Command from '../../../command.js'; -import { ERROR_TYPE, GET_STATES } from '../../../../constants/constants.js'; - -class GetLatestAssertionIdCommand extends Command { - constructor(ctx) { - super(ctx); - this.operationService = ctx.getService; - this.blockchainModuleManager = ctx.blockchainModuleManager; - this.ualService = ctx.ualService; - - this.errorType = ERROR_TYPE.GET.GET_ASSERTION_ID_ERROR; - } - - /** - * Executes command and produces one or more events - * @param command - */ - async execute(command) { - const { id, operationId, state } = command.data; - - const commandData = {}; - if (!this.ualService.isUAL(id)) { - this.handleError(operationId, `Requested id is not a UAL`, this.errorType, true); - - return Command.empty(); - } - - const { blockchain, contract, tokenId } = this.ualService.resolveUAL(id); - commandData.blockchain = blockchain; - commandData.tokenId = tokenId; - commandData.contract = contract; - - let unfinalizedAssertionId; - if (state === GET_STATES.LATEST) { - this.logger.debug( - `Searching for latest assertion id on ${blockchain} on contract: ${contract} with tokenId: ${tokenId}`, - ); - unfinalizedAssertionId = await this.blockchainModuleManager.getUnfinalizedAssertionId( - blockchain, - tokenId, - ); - commandData.assertionId = unfinalizedAssertionId; - } - - if ( - typeof unfinalizedAssertionId === 'undefined' || - !unfinalizedAssertionId || - parseInt(unfinalizedAssertionId, 16) === 0 - ) { - this.logger.debug( - `Searching for latest finalized assertion id on ${blockchain} on contract: ${contract} with tokenId: ${tokenId}`, - ); - const blockchainAssertionId = await this.blockchainModuleManager.getLatestAssertionId( - blockchain, - contract, - tokenId, - ); - if (!blockchainAssertionId) { - this.handleError( - operationId, - `Unable to find latest finalized assertion id on ${blockchain} on contract: ${contract} with tokenId: ${tokenId}`, - this.errorType, - true, - ); - - return Command.empty(); - } - commandData.assertionId = blockchainAssertionId; - } - - return this.continueSequence({ ...command.data, ...commandData }, command.sequence); - } - - async handleError(operationId, errorMessage, errorType) { - await this.operationService.markOperationAsFailed(operationId, errorMessage, errorType); - } - - /** - * Builds default getLatestAssertionIdCommand - * @param map - * @returns {{add, data: *, delay: *, deadline: *}} - */ - default(map) { - const command = { - name: 'getLatestAssertionIdCommand', - delay: 0, - transactional: false, - }; - Object.assign(command, map); - return command; - } -} - -export default GetLatestAssertionIdCommand; diff --git a/src/commands/protocols/get/sender/local-get-command.js b/src/commands/protocols/get/sender/local-get-command.js index ec3b22e76..f9d09cf0f 100644 --- a/src/commands/protocols/get/sender/local-get-command.js +++ b/src/commands/protocols/get/sender/local-get-command.js @@ -2,7 +2,6 @@ import Command from '../../../command.js'; import { OPERATION_ID_STATUS, ERROR_TYPE, - GET_STATES, TRIPLE_STORE_REPOSITORIES, PENDING_STORAGE_REPOSITORIES, } from '../../../../constants/constants.js'; @@ -10,6 +9,7 @@ import { class LocalGetCommand extends Command { constructor(ctx) { super(ctx); + this.blockchainModuleManager = ctx.blockchainModuleManager; this.config = ctx.config; this.operationService = ctx.getService; this.operationIdService = ctx.operationIdService; @@ -24,31 +24,36 @@ class LocalGetCommand extends Command { * @param command */ async execute(command) { - const { operationId, assertionId, state } = command.data; + const { operationId, blockchain, contract, tokenId, state } = command.data; await this.operationIdService.updateOperationIdStatus( operationId, OPERATION_ID_STATUS.GET.GET_LOCAL_START, ); const response = {}; - if ( - state === GET_STATES.LATEST && - command.data.blockchain != null && - command.data.contract != null && - command.data.tokenId != null - ) { - for (const repository of [ - PENDING_STORAGE_REPOSITORIES.PRIVATE, - PENDING_STORAGE_REPOSITORIES.PUBLIC, - ]) { + for (const repository of [ + PENDING_STORAGE_REPOSITORIES.PRIVATE, + PENDING_STORAGE_REPOSITORIES.PUBLIC, + ]) { + // eslint-disable-next-line no-await-in-loop + const stateIsPending = await this.pendingStorageService.assetHasPendingState( + repository, + blockchain, + contract, + tokenId, + state, + ); + + if (stateIsPending) { // eslint-disable-next-line no-await-in-loop const cachedAssertion = await this.pendingStorageService.getCachedAssertion( repository, - command.data.blockchain, - command.data.contract, - command.data.tokenId, + blockchain, + contract, + tokenId, operationId, ); + if (cachedAssertion?.public?.assertion?.length) { response.assertion = cachedAssertion.public.assertion; if (cachedAssertion?.private?.assertion?.length) { @@ -63,12 +68,11 @@ class LocalGetCommand extends Command { for (const repository of [ TRIPLE_STORE_REPOSITORIES.PRIVATE_CURRENT, TRIPLE_STORE_REPOSITORIES.PUBLIC_CURRENT, + TRIPLE_STORE_REPOSITORIES.PRIVATE_HISTORY, + TRIPLE_STORE_REPOSITORIES.PUBLIC_HISTORY, ]) { // eslint-disable-next-line no-await-in-loop - response.assertion = await this.tripleStoreService.getAssertion( - repository, - assertionId, - ); + response.assertion = await this.tripleStoreService.getAssertion(repository, state); if (response?.assertion?.length) break; } } diff --git a/src/commands/protocols/get/sender/v1.0.0/v1-0-0-get-init-command.js b/src/commands/protocols/get/sender/v1.0.0/v1-0-0-get-init-command.js index c32b436f5..ce55bf1ac 100644 --- a/src/commands/protocols/get/sender/v1.0.0/v1-0-0-get-init-command.js +++ b/src/commands/protocols/get/sender/v1.0.0/v1-0-0-get-init-command.js @@ -10,8 +10,16 @@ class GetInitCommand extends ProtocolInitCommand { } async prepareMessage(command) { - const commandData = await super.prepareMessage(command); - return { ...commandData, state: command.data.state }; + const { blockchain, contract, tokenId, keyword, assertionId, state } = command.data; + + return { + blockchain, + contract, + tokenId, + keyword, + assertionId, + state, + }; } messageTimeout() { diff --git a/src/commands/protocols/get/sender/v1.0.0/v1-0-0-get-request-command.js b/src/commands/protocols/get/sender/v1.0.0/v1-0-0-get-request-command.js index c19163f32..7c6d6e1cf 100644 --- a/src/commands/protocols/get/sender/v1.0.0/v1-0-0-get-request-command.js +++ b/src/commands/protocols/get/sender/v1.0.0/v1-0-0-get-request-command.js @@ -31,15 +31,15 @@ class GetRequestCommand extends ProtocolRequestCommand { } async prepareMessage(command) { - const { assertionId, blockchain, contract, tokenId, hashFunctionId, state } = command.data; + const { blockchain, contract, tokenId, assertionId, state, hashFunctionId } = command.data; return { - assertionId, blockchain, contract, tokenId, - hashFunctionId, + assertionId, state, + hashFunctionId, }; } diff --git a/src/commands/protocols/update/receiver/v1.0.0/v1-0-0-handle-update-request-command.js b/src/commands/protocols/update/receiver/v1.0.0/v1-0-0-handle-update-request-command.js index c26f461a4..83eff51b7 100644 --- a/src/commands/protocols/update/receiver/v1.0.0/v1-0-0-handle-update-request-command.js +++ b/src/commands/protocols/update/receiver/v1.0.0/v1-0-0-handle-update-request-command.js @@ -42,15 +42,16 @@ class HandleUpdateRequestCommand extends HandleProtocolMessageCommand { OPERATION_ID_STATUS.UPDATE.VALIDATING_UPDATE_ASSERTION_REMOTE_START, ); - const { assertion } = await this.operationIdService.getCachedOperationIdData(operationId); + const cachedData = await this.operationIdService.getCachedOperationIdData(operationId); await this.pendingStorageService.cacheAssertion( PENDING_STORAGE_REPOSITORIES.PUBLIC, blockchain, contract, tokenId, + cachedData.assertionId, { public: { - assertion, + assertion: cachedData.assertion, }, agreementId, agreementData, diff --git a/src/constants/constants.js b/src/constants/constants.js index 4f73f7bae..9c807292e 100644 --- a/src/constants/constants.js +++ b/src/constants/constants.js @@ -8,6 +8,8 @@ export const STAKE_UINT256_MULTIPLIER_BN = UINT256_MAX_BN.div(500000000); export const UINT256_UINT32_DIVISOR_BN = UINT256_MAX_BN.div(UINT32_MAX_BN); +export const ZERO_BYTES32 = `0x${'0'.repeat(64)}`; + export const SCHEMA_CONTEXT = 'http://schema.org/'; export const PRIVATE_ASSERTION_PREDICATE = @@ -35,7 +37,7 @@ export const TRIPLE_STORE_CONNECT_RETRY_FREQUENCY = 10; export const MAX_FILE_SIZE = 2621440; -export const GET_STATES = { LATEST: 'LATEST', LATEST_FINALIZED: 'LATEST_FINALIZED' }; +export const GET_STATES = { LATEST: 'LATEST', FINALIZED: 'LATEST_FINALIZED' }; export const BYTES_IN_KILOBYTE = 1024; export const BYTES_IN_MEGABYTE = BYTES_IN_KILOBYTE * BYTES_IN_KILOBYTE; diff --git a/src/controllers/http-api/get-http-api-controller.js b/src/controllers/http-api/get-http-api-controller.js index 0d5fd7eba..8855bb0a9 100644 --- a/src/controllers/http-api/get-http-api-controller.js +++ b/src/controllers/http-api/get-http-api-controller.js @@ -14,6 +14,8 @@ class GetController extends BaseController { this.operationIdService = ctx.operationIdService; this.operationService = ctx.getService; this.repositoryModuleManager = ctx.repositoryModuleManager; + this.ualService = ctx.ualService; + this.validationService = ctx.validationService; } async handleGetRequest(req, res) { @@ -38,13 +40,29 @@ class GetController extends BaseController { try { const { id } = req.body; + + if (!this.ualService.isUAL(id)) { + throw Error('Requested id is not a UAL.'); + } + + const { blockchain, contract, tokenId } = this.ualService.resolveUAL(id); + + const isValidUal = await this.validationService.validateUal( + blockchain, + contract, + tokenId, + ); + if (!isValidUal) { + throw Error(`${id} UAL isn't valid.`); + } + const state = req.body.state ?? DEFAULT_GET_STATE; const hashFunctionId = req.body.hashFunctionId ?? CONTENT_ASSET_HASH_FUNCTION_ID; this.logger.info(`Get for ${id} with operation id ${operationId} initiated.`); const commandSequence = [ - 'getLatestAssertionIdCommand', + 'getAssertionIdCommand', 'localGetCommand', 'networkGetCommand', ]; @@ -54,8 +72,10 @@ class GetController extends BaseController { sequence: commandSequence.slice(1), delay: 0, data: { + blockchain, + contract, + tokenId, operationId, - id, state, hashFunctionId, }, diff --git a/src/controllers/http-api/request-schema/get-schema.js b/src/controllers/http-api/request-schema/get-schema.js index 6874ae720..6d1d6b05a 100644 --- a/src/controllers/http-api/request-schema/get-schema.js +++ b/src/controllers/http-api/request-schema/get-schema.js @@ -7,8 +7,14 @@ export default () => ({ id: { type: 'string', }, - type: { - enum: [GET_STATES.LATEST, GET_STATES.LATEST_FINALIZED], + state: { + oneOf: [ + { enum: [GET_STATES.LATEST, GET_STATES.FINALIZED] }, + { + type: 'string', + pattern: '^0x[A-Fa-f0-9]{64}$', + }, + ], }, hashFunctionId: { type: 'number', diff --git a/src/controllers/rpc/publish-rpc-controller.js b/src/controllers/rpc/publish-rpc-controller.js index 2d02db662..84010abb3 100644 --- a/src/controllers/rpc/publish-rpc-controller.js +++ b/src/controllers/rpc/publish-rpc-controller.js @@ -30,6 +30,7 @@ class PublishController extends BaseController { // eslint-disable-next-line no-case-declarations dataSource = await this.operationIdService.getCachedOperationIdData(operationId); await this.operationIdService.cacheOperationIdData(operationId, { + assertionId: dataSource.assertionId, assertion: message.data.assertion, }); command.name = handleRequestCommand; diff --git a/src/controllers/rpc/update-rpc-controller.js b/src/controllers/rpc/update-rpc-controller.js index 7cc6a795c..f9fd8764d 100644 --- a/src/controllers/rpc/update-rpc-controller.js +++ b/src/controllers/rpc/update-rpc-controller.js @@ -1,5 +1,8 @@ import BaseController from './base-rpc-controller.js'; -import { CONTENT_ASSET_HASH_FUNCTION_ID, NETWORK_MESSAGE_TYPES } from '../../constants/constants.js'; +import { + CONTENT_ASSET_HASH_FUNCTION_ID, + NETWORK_MESSAGE_TYPES, +} from '../../constants/constants.js'; class UpdateController extends BaseController { constructor(ctx) { @@ -27,6 +30,7 @@ class UpdateController extends BaseController { // eslint-disable-next-line no-case-declarations dataSource = await this.operationIdService.getCachedOperationIdData(operationId); await this.operationIdService.cacheOperationIdData(operationId, { + assertionId: dataSource.assertionId, assertion: message.data.assertion, }); command.name = handleRequestCommand; diff --git a/src/migration/base-migration.js b/src/migration/base-migration.js index 3755f03a5..0b17fa657 100644 --- a/src/migration/base-migration.js +++ b/src/migration/base-migration.js @@ -27,7 +27,7 @@ class BaseMigration { this.fileService.getMigrationFolderPath(), this.migrationName, ); - if (await this.fileService.fileExists(migrationFilePath)) { + if (await this.fileService.pathExists(migrationFilePath)) { return true; } return false; diff --git a/src/migration/blockchain-identity-migration.js b/src/migration/blockchain-identity-migration.js index 6086ab61a..fb697a1ee 100644 --- a/src/migration/blockchain-identity-migration.js +++ b/src/migration/blockchain-identity-migration.js @@ -15,7 +15,7 @@ class BlockchainIdentityMigration extends BaseMigration { this.config.configFilename, ); - const config = await this.fileService.loadJsonFromFile(configurationFilePath); + const config = await this.fileService.readFile(configurationFilePath, true); for (const blockchainImpl in config.modules.blockchain.implementation) { delete config.modules.blockchain.implementation[blockchainImpl].config.identity; } diff --git a/src/migration/pending-storage-migration.js b/src/migration/pending-storage-migration.js new file mode 100644 index 000000000..040cf491a --- /dev/null +++ b/src/migration/pending-storage-migration.js @@ -0,0 +1,38 @@ +import path from 'path'; +import { calculateRoot } from 'assertion-tools'; +import { PENDING_STORAGE_REPOSITORIES } from '../constants/constants.js'; +import BaseMigration from './base-migration.js'; + +class PendingStorageMigration extends BaseMigration { + async executeMigration() { + const promises = Object.values(PENDING_STORAGE_REPOSITORIES).map(async (repository) => { + let fileNames; + const repositoryPath = this.fileService.getPendingStorageCachePath(repository); + try { + fileNames = await this.fileService.readDirectory(repositoryPath); + } catch (error) { + return false; + } + + await Promise.all( + fileNames.map(async (fileName) => { + const newDirectoryPath = path.join(repositoryPath, fileName); + const cachedData = await this.fileService.readFile(newDirectoryPath, true); + await this.fileService.removeFile(newDirectoryPath); + if (cachedData?.public?.assertion) { + const newDocumentName = calculateRoot(cachedData.public.assertion); + await this.fileService.writeContentsToFile( + newDirectoryPath, + newDocumentName, + JSON.stringify(cachedData), + ); + } + }), + ); + }); + + await Promise.all(promises); + } +} + +export default PendingStorageMigration; diff --git a/src/migration/service-agreements-metadata-migration.js b/src/migration/service-agreements-metadata-migration.js index bcfbbeb9b..8c29a8771 100644 --- a/src/migration/service-agreements-metadata-migration.js +++ b/src/migration/service-agreements-metadata-migration.js @@ -32,9 +32,9 @@ class ServiceAgreementsMetadataMigration extends BaseMigration { const migrationInfoFileName = `${this.migrationName}_info`; const migrationInfoPath = path.join(migrationFolderPath, migrationInfoFileName); let migrationInfo; - if (await this.fileService.fileExists(migrationInfoPath)) { + if (await this.fileService.pathExists(migrationInfoPath)) { migrationInfo = await this.fileService - ._readFile(migrationInfoPath, true) + .readFile(migrationInfoPath, true) .catch(() => {}); } if (!migrationInfo?.lastProcessedTokenId) { diff --git a/src/migration/triple-store-metadata-migration.js b/src/migration/triple-store-metadata-migration.js index ffdcc87cc..dc087689d 100644 --- a/src/migration/triple-store-metadata-migration.js +++ b/src/migration/triple-store-metadata-migration.js @@ -36,9 +36,9 @@ class TripleStoreMetadataMigration extends BaseMigration { const migrationInfoPath = path.join(migrationFolderPath, migrationInfoFileName); let migrationInfo; - if (await this.fileService.fileExists(migrationInfoPath)) { + if (await this.fileService.pathExists(migrationInfoPath)) { try { - migrationInfo = await this.fileService._readFile(migrationInfoPath, true); + migrationInfo = await this.fileService.readFile(migrationInfoPath, true); } catch (error) { migrationInfo = { status: 'IN_PROGRESS', @@ -82,9 +82,9 @@ class TripleStoreMetadataMigration extends BaseMigration { const migrationInfoFileName = `${this.migrationName}_${currentRepository}`; const migrationInfoPath = path.join(migrationFolderPath, migrationInfoFileName); let migrationInfo; - if (await this.fileService.fileExists(migrationInfoPath)) { + if (await this.fileService.pathExists(migrationInfoPath)) { try { - migrationInfo = await this.fileService._readFile(migrationInfoPath, true); + migrationInfo = await this.fileService.readFile(migrationInfoPath, true); } catch (error) { migrationInfo = { status: 'IN_PROGRESS', diff --git a/src/migration/triple-store-user-configuration-migration.js b/src/migration/triple-store-user-configuration-migration.js index cb0c2c258..ba6f7b864 100644 --- a/src/migration/triple-store-user-configuration-migration.js +++ b/src/migration/triple-store-user-configuration-migration.js @@ -15,9 +15,7 @@ class TripleStoreUserConfigurationMigration extends BaseMigration { this.config.configFilename, ); - const userConfiguration = await this.fileService.loadJsonFromFile( - configurationFilePath, - ); + const userConfiguration = await this.fileService.readFile(configurationFilePath, true); if (userConfiguration.modules.tripleStore.implementation) { for (const implementationName in userConfiguration.modules.tripleStore .implementation) { @@ -91,8 +89,9 @@ class TripleStoreUserConfigurationMigration extends BaseMigration { 'local-network-setup', '.origintrail_noderc_template.json', ); - const configurationTemplate = await this.fileService.loadJsonFromFile( + const configurationTemplate = await this.fileService.readFile( configurationTemplatePath, + true, ); if ( diff --git a/src/modules/blockchain/blockchain-module-manager.js b/src/modules/blockchain/blockchain-module-manager.js index 53c32c0d9..55ea81e81 100644 --- a/src/modules/blockchain/blockchain-module-manager.js +++ b/src/modules/blockchain/blockchain-module-manager.js @@ -115,6 +115,13 @@ class BlockchainModuleManager extends BaseModuleManager { ]); } + async getKnowledgeAssetOwner(blockchain, assetContractAddress, tokenId) { + return this.callImplementationFunction(blockchain, 'getKnowledgeAssetOwner', [ + assetContractAddress, + tokenId, + ]); + } + async getUnfinalizedAssertionId(blockchain, tokenId) { return this.callImplementationFunction(blockchain, 'getUnfinalizedState', [tokenId]); } diff --git a/src/modules/blockchain/implementation/web3-service.js b/src/modules/blockchain/implementation/web3-service.js index 946a56202..6a30def3b 100644 --- a/src/modules/blockchain/implementation/web3-service.js +++ b/src/modules/blockchain/implementation/web3-service.js @@ -17,7 +17,7 @@ import { const require = createRequire(import.meta.url); const ABIs = { - AbstractAsset: require('dkg-evm-module/abi/AbstractAsset.json'), + ContentAssetStorage: require('dkg-evm-module/abi/ContentAssetStorage.json'), AssertionStorage: require('dkg-evm-module/abi/AssertionStorage.json'), Staking: require('dkg-evm-module/abi/Staking.json'), StakingStorage: require('dkg-evm-module/abi/StakingStorage.json'), @@ -171,7 +171,7 @@ class Web3Service { initializeAssetStorageContract(assetStorageAddress) { this.assetStorageContracts[assetStorageAddress.toLowerCase()] = new ethers.Contract( assetStorageAddress, - ABIs.AbstractAsset, + ABIs.ContentAssetStorage, this.wallet, ); } @@ -525,6 +525,14 @@ class Web3Service { ]); } + async getKnowledgeAssetOwner(assetContractAddress, tokenId) { + const assetStorageContractInstance = + this.assetStorageContracts[assetContractAddress.toString().toLowerCase()]; + if (!assetStorageContractInstance) throw Error('Unknown asset storage contract address'); + + return this.callContractFunction(assetStorageContractInstance, 'ownerOf', [tokenId]); + } + async getUnfinalizedState(tokenId) { return this.callContractFunction( this.UnfinalizedStateStorageContract, diff --git a/src/service/file-service.js b/src/service/file-service.js index 847bfada0..aa1c5be91 100644 --- a/src/service/file-service.js +++ b/src/service/file-service.js @@ -1,5 +1,5 @@ import path from 'path'; -import { mkdir, writeFile, readFile, unlink, stat, readdir } from 'fs/promises'; +import { mkdir, writeFile, readFile, unlink, stat, readdir, rm } from 'fs/promises'; import appRootPath from 'app-root-path'; const MIGRATION_FOLDER_NAME = 'migrations'; @@ -12,8 +12,8 @@ class FileService { this.logger = ctx.logger; } - getFileExtension(fileName) { - return path.extname(fileName).toLowerCase(); + getFileExtension(filePath) { + return path.extname(filePath).toLowerCase(); } /** @@ -25,7 +25,7 @@ class FileService { */ async writeContentsToFile(directory, filename, data, log = true) { if (log) { - this.logger.debug(`Saving file with name: ${filename} in directory: ${directory}`); + this.logger.debug(`Saving file with name: ${filename} in the directory: ${directory}`); } await mkdir(directory, { recursive: true }); const fullpath = path.join(directory, filename); @@ -33,56 +33,75 @@ class FileService { return fullpath; } - readFileOnPath(filePath) { - return this._readFile(filePath, false); - } - async readDirectory(dirPath) { - return readdir(dirPath); + this.logger.debug(`Reading folder at path: ${dirPath}`); + try { + return readdir(dirPath); + } catch (error) { + if (error.code === 'ENOENT') { + throw Error(`Folder not found at path: ${dirPath}`); + } + throw error; + } } async stat(filePath) { return stat(filePath); } - /** - * Loads JSON data from file - * @returns {Promise} - * @private - */ - loadJsonFromFile(filePath) { - return this._readFile(filePath, true); - } - - async fileExists(filePath) { + async pathExists(fileOrDirPath) { try { - await stat(filePath); + await stat(fileOrDirPath); return true; - } catch (e) { - return false; + } catch (error) { + if (error.code === 'ENOENT') { + return false; + } + throw error; } } - async _readFile(filePath, convertToJSON = false) { - this.logger.debug( - `Reading file on path: ${filePath}, converting to json: ${convertToJSON}`, - ); + async readFile(filePath, convertToJSON = false) { + this.logger.debug(`Reading file: ${filePath}, converting to json: ${convertToJSON}`); try { const data = await readFile(filePath); return convertToJSON ? JSON.parse(data) : data.toString(); - } catch (e) { - throw Error(`File not found on path: ${filePath}`); + } catch (error) { + if (error.code === 'ENOENT') { + throw Error(`File not found at path: ${filePath}`); + } + throw error; } } async removeFile(filePath) { - if (await this.fileExists(filePath)) { - this.logger.trace(`Removing file on path: ${filePath}`); + this.logger.trace(`Removing file at path: ${filePath}`); + + try { await unlink(filePath); return true; + } catch (error) { + if (error.code === 'ENOENT') { + this.logger.debug(`File not found at path: ${filePath}`); + return false; + } + throw error; + } + } + + async removeFolder(folderPath) { + // this.logger.trace(`Removing folder at path: ${folderPath}`); + + try { + await rm(folderPath, { recursive: true }); + return true; + } catch (error) { + if (error.code === 'ENOENT') { + this.logger.debug(`Folder not found at path: ${folderPath}`); + return false; + } + throw error; } - this.logger.debug(`File not found on path: ${filePath}`); - return false; } getDataFolderPath() { @@ -108,21 +127,35 @@ class FileService { return path.join(this.getOperationIdCachePath(), operationId); } - getPendingStorageFileName(blockchain, contract, tokenId) { - return `${blockchain.toLowerCase()}:${contract.toLowerCase()}:${tokenId}`; - } - getPendingStorageCachePath(repository) { return path.join(this.getDataFolderPath(), 'pending_storage_cache', repository); } - getPendingStorageDocumentPath(repository, blockchain, contract, tokenId) { + getPendingStorageFolderPath(repository, blockchain, contract, tokenId) { return path.join( this.getPendingStorageCachePath(repository), - this.getPendingStorageFileName(blockchain, contract, tokenId), + `${blockchain.toLowerCase()}:${contract.toLowerCase()}:${tokenId}`, ); } + async getPendingStorageDocumentPath(repository, blockchain, contract, tokenId, assertionId) { + const pendingStorageFolder = this.getPendingStorageFolderPath( + repository, + blockchain, + contract, + tokenId, + ); + + let pendingStorageFileName; + if (assertionId === undefined) { + [pendingStorageFileName] = await this.readDirectory(pendingStorageFolder); + } else { + pendingStorageFileName = assertionId; + } + + return path.join(pendingStorageFolder, pendingStorageFileName); + } + getArchiveFolderPath(subFolder) { return path.join(this.getDataFolderPath(), ARCHIVE_FOLDER_NAME, subFolder); } diff --git a/src/service/operation-id-service.js b/src/service/operation-id-service.js index e882317e9..3e714c2a7 100644 --- a/src/service/operation-id-service.js +++ b/src/service/operation-id-service.js @@ -113,8 +113,8 @@ class OperationIdService { this.logger.debug(`Reading operation id: ${operationId} cached data from file`); const documentPath = this.fileService.getOperationIdDocumentPath(operationId); let data; - if (await this.fileService.fileExists(documentPath)) { - data = await this.fileService.loadJsonFromFile(documentPath); + if (await this.fileService.pathExists(documentPath)) { + data = await this.fileService.readFile(documentPath, true); } return data; } @@ -146,7 +146,7 @@ class OperationIdService { async removeExpiredOperationIdFileCache(expiredTimeout, batchSize) { const cacheFolderPath = this.fileService.getOperationIdCachePath(); - const cacheFolderExists = await this.fileService.fileExists(cacheFolderPath); + const cacheFolderExists = await this.fileService.pathExists(cacheFolderPath); if (!cacheFolderExists) { return; } diff --git a/src/service/pending-storage-service.js b/src/service/pending-storage-service.js index 700b3d345..18d1a94e1 100644 --- a/src/service/pending-storage-service.js +++ b/src/service/pending-storage-service.js @@ -1,28 +1,35 @@ class PendingStorageService { constructor(ctx) { this.logger = ctx.logger; - this.fileService = ctx.fileService; this.ualService = ctx.ualService; } - async cacheAssertion(repository, blockchain, contract, tokenId, assertion, operationId) { + async cacheAssertion( + repository, + blockchain, + contract, + tokenId, + assertionId, + assertion, + operationId, + ) { const ual = this.ualService.deriveUAL(blockchain, contract, tokenId); this.logger.debug( - `Caching assertion for ual: ${ual}, operation id: ${operationId} in file in ${repository} pending storage`, + `Caching ${assertionId} assertion for ual: ${ual}, operation id: ${operationId} in file in ${repository} pending storage`, ); - const documentPath = this.fileService.getPendingStorageCachePath(repository); - const documentName = this.fileService.getPendingStorageFileName( + const pendingStorageFolderPath = this.fileService.getPendingStorageFolderPath( + repository, blockchain, contract, tokenId, ); await this.fileService.writeContentsToFile( - documentPath, - documentName, + pendingStorageFolderPath, + assertionId, JSON.stringify(assertion), ); } @@ -33,19 +40,20 @@ class PendingStorageService { this.logger.debug( `Reading cached assertion for ual: ${ual}, operation id: ${operationId} from file in ${repository} pending storage`, ); + try { + const documentPath = await this.fileService.getPendingStorageDocumentPath( + repository, + blockchain, + contract, + tokenId, + ); - const documentPath = this.fileService.getPendingStorageDocumentPath( - repository, - blockchain, - contract, - tokenId, - ); - let data; - if (await this.fileService.fileExists(documentPath)) { - data = await this.fileService.loadJsonFromFile(documentPath); + const data = await this.fileService.readFile(documentPath, true); + return data; + } catch (error) { + this.logger.debug('Assertion not found in pending storage'); + return null; } - - return data; } async removeCachedAssertion(repository, blockchain, contract, tokenId, operationId) { @@ -55,26 +63,30 @@ class PendingStorageService { `Removing cached assertion for ual: ${ual} operation id: ${operationId} from file in ${repository} pending storage`, ); - const documentPath = this.fileService.getPendingStorageDocumentPath( + const pendingStorageFolderPath = this.fileService.getPendingStorageFolderPath( repository, blockchain, contract, - tokenId, ); - await this.fileService.removeFile(documentPath); + await this.fileService.removeFolder(pendingStorageFolderPath); } - async assertionExists(repository, blockchain, contract, tokenId) { - const documentPath = this.fileService.getPendingStorageDocumentPath( - repository, - blockchain, - contract, - tokenId, - ); - this.logger.trace( - `Checking if assertion exists in pending storage on path: ${documentPath}`, - ); - return this.fileService.fileExists(documentPath); + async assetHasPendingState(repository, blockchain, contract, tokenId, assertionId) { + try { + const documentPath = await this.fileService.getPendingStorageDocumentPath( + repository, + blockchain, + contract, + tokenId, + assertionId, + ); + this.logger.trace( + `Checking if assertion exists in pending storage at path: ${documentPath}`, + ); + return this.fileService.pathExists(documentPath); + } catch (error) { + return false; + } } } diff --git a/src/service/validation-service.js b/src/service/validation-service.js index ceaa96f86..d74700b89 100644 --- a/src/service/validation-service.js +++ b/src/service/validation-service.js @@ -7,6 +7,25 @@ class ValidationService { this.blockchainModuleManager = ctx.blockchainModuleManager; } + async validateUal(blockchain, contract, tokenId) { + this.logger.info( + `Validating UAL: did:dkg:${blockchain.toLowerCase()}/${contract.toLowerCase()}/${tokenId}`, + ); + + let isValid = true; + try { + await this.blockchainModuleManager.getKnowledgeAssetOwner( + blockchain, + contract, + tokenId, + ); + } catch (err) { + isValid = false; + } + + return isValid; + } + async validateAssertion(assertionId, blockchain, assertion) { this.logger.info(`Validating assertionId: ${assertionId}`); diff --git a/test/bdd/features/get-errors.feature b/test/bdd/features/get-errors.feature index 9ec5db490..27b8516da 100644 --- a/test/bdd/features/get-errors.feature +++ b/test/bdd/features/get-errors.feature @@ -4,23 +4,44 @@ Feature: Get errors test And 1 bootstrap is running @get-errors - Scenario: Getting non existent UAL + Scenario: Getting non-existent UAL + Given I setup 4 nodes + And I wait for 2 seconds + + When I call Get directly on the node 1 with nonExistentUAL + And I wait for latest resolve to finalize + Then Latest Get operation finished with status: GetRouteError + + @get-errors + Scenario: Getting invalid UAL Given I setup 4 nodes And I wait for 2 seconds - And I call get directly to ot-node 1 with nonExistentUAL - And I wait for last resolve to finalize - Then Last Get operation finished with status: GetAssertionIdError - #@get-errors - #Scenario: GET operation result on a node with minimum replication factor greater than the number of nodes - #Given I setup 4 nodes - #And I wait for 2 seconds - #And I call publish on node 1 with validAssertion - #Then Last PUBLISH operation finished with status: COMPLETED - #When I setup node 5 with minimumAckResponses.get set to 10 - #And I wait for 2 seconds - #And I get operation result from node 5 for last published assertion - #And I wait for last resolve to finalize - #Then Last GET operation finished with status: GetNetworkError + When I call Get directly on the node 1 with invalidUAL + And I wait for latest resolve to finalize + Then Latest Get operation finished with status: GetRouteError + @get-errors + Scenario: Getting non-existent state + Given I setup 4 nodes + And I set R0 to be 1 + And I set R1 to be 2 + And I wait for 2 seconds + + When I call Publish on the node 1 with validAssertion + And I wait for latest Publish to finalize + And I call Get directly on the node 1 with nonExistentState + Then It should fail with status code 400 + + @get-errors + Scenario: Getting invalid state hash + Given I setup 4 nodes + And I set R0 to be 1 + And I set R1 to be 2 + And I wait for 2 seconds + When I call Publish on the node 1 with validAssertion + And I wait for latest Publish to finalize + And I call Get directly on the node 1 with invalidStateHash + And I wait for latest resolve to finalize + Then Latest Get operation finished with status: GetAssertionIdError diff --git a/test/bdd/features/get.feature b/test/bdd/features/get.feature new file mode 100644 index 000000000..c6e994b21 --- /dev/null +++ b/test/bdd/features/get.feature @@ -0,0 +1,76 @@ +Feature: Get asset states test + Background: Setup local blockchain, bootstraps and nodes + Given the blockchain is set up + And 1 bootstrap is running + + @release + Scenario: Get first state of the updated knowledge asset + Given I set R0 to be 1 + And I set R1 to be 2 + And I set finalizationCommitsNumber to be 2 + And I setup 4 nodes + And I wait for 2 seconds + + When I call Publish on the node 4 with validPublish_1ForValidUpdate_1 + And I wait for latest Publish to finalize + Then Latest Publish operation finished with status: COMPLETED + + When I call Update on the node 4 for the latest published UAL with validUpdate_1 + And I wait for latest Update to finalize + Then Latest Update operation finished with status: COMPLETED + + When I call Get directly on the node 4 with validGetFirstStateRequestBody + And I wait for latest resolve to finalize + Then Latest Get operation finished with status: COMPLETED + + @release + Scenario: Get latest state of the updated knowledge asset + Given I set R0 to be 1 + And I set R1 to be 2 + And I set finalizationCommitsNumber to be 2 + And I setup 4 nodes + And I wait for 2 seconds + + When I call Publish on the node 4 with validPublish_1ForValidUpdate_1 + And I wait for latest Publish to finalize + Then Latest Publish operation finished with status: COMPLETED + + When I call Update on the node 4 for the latest published UAL with validUpdate_1 + And I wait for latest Update to finalize + Then Latest Update operation finished with status: COMPLETED + + When I call Get directly on the node 4 with validGetUpdatedStateRequestBody + And I wait for latest resolve to finalize + Then Latest Get operation finished with status: COMPLETED + + @release + Scenario: Get all states of the knowledge asset that is updated 2 times + Given I set R0 to be 1 + And I set R1 to be 2 + And I set finalizationCommitsNumber to be 2 + And I setup 4 nodes + And I wait for 2 seconds + + When I call Publish on the node 4 with validPublish_1ForValidUpdate_1 + And I wait for latest Publish to finalize + Then Latest Publish operation finished with status: COMPLETED + + When I call Update on the node 4 for the latest published UAL with validUpdate_1 + And I wait for latest Update to finalize + Then Latest Update operation finished with status: COMPLETED + + When I call Update on the node 4 for the latest published UAL with validUpdate_2 + And I wait for latest Update to finalize + Then Latest Update operation finished with status: COMPLETED + + When I call Get directly on the node 4 with getFirstStateRequestBody + And I wait for latest resolve to finalize + Then Latest Get operation finished with status: COMPLETED + + When I call Get directly on the node 4 with getSecondStateRequestBody + And I wait for latest resolve to finalize + Then Latest Get operation finished with status: COMPLETED + + When I call Get directly on the node 4 with getThirdStateRequestBody + And I wait for latest resolve to finalize + Then Latest Get operation finished with status: COMPLETED diff --git a/test/bdd/features/publish-errors.feature b/test/bdd/features/publish-errors.feature index bf6754594..47c62187f 100644 --- a/test/bdd/features/publish-errors.feature +++ b/test/bdd/features/publish-errors.feature @@ -3,26 +3,20 @@ Feature: Publish errors test Given the blockchain is set up And 1 bootstrap is running - #@publish-errors - #Scenario: Publish on a node with invalid data path - #Given I setup 3 nodes - #And I setup node 4 with appDataPath set to \0 - #And I wait for 2 seconds - #And I call publish on node 4 with validAssertion - #Then Last PUBLISH operation finished with status: PublishRouteError - @publish-errors Scenario: Publish on a node with minimum replication factor greater than the number of nodes Given I setup 1 nodes - And I call publish on node 1 with validAssertion - Then Last Publish operation finished with status: PublishStartError + And I wait for 2 seconds + + When I call Publish on the node 1 with validAssertion + And I wait for latest Publish to finalize + Then Latest Publish operation finished with status: PublishStartError @publish-errors - Scenario: Publish an asset directly on the node + Scenario: Publish a knowledge asset directly on the node Given I setup 1 nodes - And I call publish on ot-node 1 directly with validPublishRequestBody - And I wait for last publish to finalize - Then Last Publish operation finished with status: ValidateAssetError -# -# + And I wait for 2 seconds + When I call Publish directly on the node 1 with validPublishRequestBody + And I wait for latest Publish to finalize + Then Latest Publish operation finished with status: ValidateAssetError diff --git a/test/bdd/features/publish.feature b/test/bdd/features/publish.feature index b38832ad5..0e28c9c9e 100644 --- a/test/bdd/features/publish.feature +++ b/test/bdd/features/publish.feature @@ -6,9 +6,10 @@ Feature: Release related tests @release Scenario: Publishing a valid assertion Given I set R0 to be 1 - Given I set R1 to be 2 - Given I setup 4 nodes - And I wait for 10 seconds + And I set R1 to be 2 + And I setup 4 nodes + And I wait for 2 seconds - When I call publish on node 4 with validAssertion - Then Last Publish operation finished with status: COMPLETED + When I call Publish on the node 4 with validAssertion + And I wait for latest Publish to finalize + Then Latest Publish operation finished with status: COMPLETED diff --git a/test/bdd/features/update-errors.feature b/test/bdd/features/update-errors.feature index 794ea018d..6633eca3f 100644 --- a/test/bdd/features/update-errors.feature +++ b/test/bdd/features/update-errors.feature @@ -4,9 +4,11 @@ Feature: Update errors test And 1 bootstrap is running @update-errors - Scenario: Update asset that was not previously published + Scenario: Update knowledge asset that was not previously published Given I setup 1 node - And I call update on ot-node 1 directly with validUpdateRequestBody - And I wait for last update to finalize - Then Last Update operation finished with status: ValidateAssetError + And I wait for 2 seconds + + When I call Update directly on the node 1 with validUpdateRequestBody + And I wait for latest Update to finalize + Then Latest Update operation finished with status: ValidateAssetError diff --git a/test/bdd/features/update.feature b/test/bdd/features/update.feature index 748c48013..a70fd206c 100644 --- a/test/bdd/features/update.feature +++ b/test/bdd/features/update.feature @@ -4,12 +4,17 @@ Feature: Update asset test And 1 bootstrap is running @release - Scenario: Update an existing asset + Scenario: Update an existing knowledge asset Given I set R0 to be 1 - Given I set R1 to be 2 - Given I setup 4 nodes - And I wait for 10 seconds - When I call publish on node 4 with validPublish_1ForValidUpdate_1 - Then Last Publish operation finished with status: COMPLETED - Given I call update on node 4 for last publish UAL with validUpdate_1 - When Last Update operation finished with status: COMPLETED + And I set R1 to be 2 + And I set finalizationCommitsNumber to be 2 + And I setup 4 nodes + And I wait for 2 seconds + + When I call Publish on the node 4 with validPublish_1ForValidUpdate_1 + And I wait for latest Publish to finalize + Then Latest Publish operation finished with status: COMPLETED + + When I call Update on the node 4 for the latest published UAL with validUpdate_1 + And I wait for latest Update to finalize + Then Latest Update operation finished with status: COMPLETED diff --git a/test/bdd/steps/api/datasets/assertions.json b/test/bdd/steps/api/datasets/assertions.json index 9f5115b08..57c923b3c 100644 --- a/test/bdd/steps/api/datasets/assertions.json +++ b/test/bdd/steps/api/datasets/assertions.json @@ -61,5 +61,20 @@ "@id": "uuid:Nis" } } + }, + "validUpdate_2": { + "public": { + "@context": [ + "https://schema.org" + ], + "@id": "uuid:3", + "company": "TL", + "user": { + "@id": "uuid:user:3" + }, + "city": { + "@id": "uuid:Funchal" + } + } } } diff --git a/test/bdd/steps/api/datasets/requests.json b/test/bdd/steps/api/datasets/requests.json index 66e7e99bf..be7258046 100644 --- a/test/bdd/steps/api/datasets/requests.json +++ b/test/bdd/steps/api/datasets/requests.json @@ -9,9 +9,27 @@ "_:c14n0 ." ], "blockchain": "hardhat", - "contract": "0x5FbDB2315678afecb367f032d93F642f64180aa3", - "tokenId": 0, - "hashFunctionId": 1 + "contract": "0xB0D4afd8879eD9F52b28595d31B441D079B2Ca07", + "tokenId": 0 + }, + "validGetFirstStateRequestBody": { + "id": "did:dkg:hardhat/0xB0D4afd8879eD9F52b28595d31B441D079B2Ca07/0", + "state": "0xe3a6733d7b999ca6f0d141afe3e38ac59223a4dfde7a5458932d2094ed4193cf" + }, + "validGetUpdatedStateRequestBody": { + "id": "did:dkg:hardhat/0xB0D4afd8879eD9F52b28595d31B441D079B2Ca07/0" + }, + "getFirstStateRequestBody": { + "id": "did:dkg:hardhat/0xB0D4afd8879eD9F52b28595d31B441D079B2Ca07/0", + "state": "0xe3a6733d7b999ca6f0d141afe3e38ac59223a4dfde7a5458932d2094ed4193cf" + }, + "getSecondStateRequestBody": { + "id": "did:dkg:hardhat/0xB0D4afd8879eD9F52b28595d31B441D079B2Ca07/0", + "state": "0x591503a1c8ba4667dd7afd203025c1bf594d817d8eec71274fe960d69fb8584f" + }, + "getThirdStateRequestBody": { + "id": "did:dkg:hardhat/0xB0D4afd8879eD9F52b28595d31B441D079B2Ca07/0", + "state": "0x759c285786b95622dad67a6be857a4eec3d9ba0caec991ed4297629ae6abbc0d" }, "blockchainNotDefinedRequestBody": { "publishType": "asset", @@ -24,11 +42,22 @@ "_:c14n0 ." ], "blockchain": null, - "contract": "0x791ee543738B997B7A125bc849005B62aFD35578", + "contract": "0xB0D4afd8879eD9F52b28595d31B441D079B2Ca07", "tokenId": 0 }, "nonExistentUAL": { - "id": "did:ganache:0x791ee543738B997B7A125bc849005B62aFD35578/1" + "id": "did:dkg:hardhat/0xB0D4afd8879eD9F52b28595d31B441D079B2Ca07/1" + }, + "invalidUAL": { + "id": "did:dkg:hardhat/0xB0D4afd8879eD9F/52b28595d31B441D079B2Ca07/1" + }, + "nonExistentState": { + "id": "did:dkg:hardhat/0xB0D4afd8879eD9F52b28595d31B441D079B2Ca07/0", + "state": -1 + }, + "invalidStateHash": { + "id": "did:dkg:hardhat/0xB0D4afd8879eD9F52b28595d31B441D079B2Ca07/0", + "state": "0x591503a1c8ba4667dd7afd203025c1bf594d817d8eec71274fe960d69fb8584e" }, "validUpdateRequestBody": { "assertionId": "0x591503a1c8ba4667dd7afd203025c1bf594d817d8eec71274fe960d69fb8584f", @@ -38,8 +67,7 @@ " ." ], "blockchain": "hardhat", - "contract": "0xb7278A61aa25c888815aFC32Ad3cC52fF24fE575", - "tokenId": 0, - "hashFunctionId": 1 + "contract": "0xB0D4afd8879eD9F52b28595d31B441D079B2Ca07", + "tokenId": 0 } } diff --git a/test/bdd/steps/api/get.mjs b/test/bdd/steps/api/get.mjs new file mode 100644 index 000000000..47ea8df7a --- /dev/null +++ b/test/bdd/steps/api/get.mjs @@ -0,0 +1,105 @@ +import { Then, When } from '@cucumber/cucumber'; +import { expect, assert } from 'chai'; +import { readFile } from 'fs/promises'; +import HttpApiHelper from '../../../utilities/http-api-helper.mjs'; + +const requests = JSON.parse(await readFile('test/bdd/steps/api/datasets/requests.json')); + +const httpApiHelper = new HttpApiHelper(); + +When( + /^I call Get on the node (\d+) for state index (\d+)/, + { timeout: 120000 }, + async function get(node, stateIndex) { + this.logger.log(`I call get route on the node ${node} for state index ${stateIndex}.`); + + const { UAL } = this.state.latestUpdateData; + const result = await this.state.nodes[node - 1].client + .getHistorical(UAL, stateIndex) + .catch((error) => { + assert.fail(`Error while trying to update assertion. ${error}`); + }); + const { operationId } = result.operation; + this.state.latestUpdateData = { + nodeId: node - 1, + operationId, + }; + }, +); + +When( + /^I call Get directly on the node (\d+) with ([^"]*)/, + { timeout: 30000 }, + async function getFromNode(node, requestName) { + this.logger.log(`I call get directly on the node ${node}`); + expect( + !!requests[requestName], + `Request body with name: ${requestName} not found!`, + ).to.be.equal(true); + const requestBody = requests[requestName]; + + try { + const result = await httpApiHelper.get(this.state.nodes[node - 1].nodeRpcUrl, requestBody); + const { operationId } = result.data; + this.state.latestGetData = { + nodeId: node - 1, + operationId, + }; + } catch (error) { + this.state.latestError = error; + } + }, +); + +Then( + /^It should fail with status code (\d+)/, + function checkLatestError(expectedStatusCode) { + const expectedStatusCodeInt = parseInt(expectedStatusCode, 10); + assert( + this.state.latestError, + 'No error occurred' + ); + assert( + this.state.latestError.statusCode, + 'No status code in error' + ); + assert( + this.state.latestError.statusCode === expectedStatusCodeInt, + `Expected request to fail with status code ${expectedStatusCodeInt}, but it failed with another code.` + ); + }, +); + +When('I wait for latest Get to finalize', { timeout: 80000 }, async function getFinalize() { + this.logger.log('I wait for latest get to finalize'); + expect( + !!this.state.latestGetData, + 'Latest get data is undefined. Get was not started.', + ).to.be.equal(true); + const getData = this.state.latestGetData; + let retryCount = 0; + const maxRetryCount = 5; + for (retryCount = 0; retryCount < maxRetryCount; retryCount += 1) { + this.logger.log( + `Getting get result for operation id: ${getData.operationId} on the node: ${getData.nodeId}`, + ); + // eslint-disable-next-line no-await-in-loop + const getResult = await httpApiHelper.getOperationResult( + this.state.nodes[getData.nodeId].nodeRpcUrl, + 'get', + getData.operationId, + ); + this.logger.log(`Operation status: ${getResult.data.status}`); + if (['COMPLETED', 'FAILED'].includes(getResult.data.status)) { + this.state.latestGetData.result = getResult; + this.state.latestGetData.status = getResult.data.status; + this.state.latestGetData.errorType = getResult.data.data?.errorType; + break; + } + if (retryCount === maxRetryCount - 1) { + assert.fail('Unable to fetch get result'); + } + // eslint-disable-next-line no-await-in-loop + await setTimeout(4000); + } +}); diff --git a/test/bdd/steps/api/info.mjs b/test/bdd/steps/api/info.mjs index 039a83450..017b4be04 100644 --- a/test/bdd/steps/api/info.mjs +++ b/test/bdd/steps/api/info.mjs @@ -3,9 +3,9 @@ import assert from 'assert'; let info = {}; -When(/^I call info route on node (\d+)/, { timeout: 120000 }, async function infoRouteCall(node) { +When(/^I call Info route on the node (\d+)/, { timeout: 120000 }, async function infoRouteCall(node) { // todo validate node number - this.logger.log('I call info route on node: ', node); + this.logger.log(`I call info route on the node ${node}`); info = await this.state.nodes[node - 1].client.info(); }); diff --git a/test/bdd/steps/api/publish.mjs b/test/bdd/steps/api/publish.mjs index ec7dcab97..87a3d5ab5 100644 --- a/test/bdd/steps/api/publish.mjs +++ b/test/bdd/steps/api/publish.mjs @@ -1,4 +1,4 @@ -import { When, Given } from '@cucumber/cucumber'; +import { When } from '@cucumber/cucumber'; import { expect, assert } from 'chai'; import { setTimeout } from 'timers/promises'; import { readFile } from 'fs/promises'; @@ -10,25 +10,23 @@ const requests = JSON.parse(await readFile('test/bdd/steps/api/datasets/requests const httpApiHelper = new HttpApiHelper(); When( - /^I call publish on node (\d+) with ([^"]*)/, + /^I call Publish on the node (\d+) with ([^"]*)/, { timeout: 120000 }, async function publish(node, assertionName) { - this.logger.log(`I call publish route on node ${node}`); + this.logger.log(`I call publish route on the node ${node}`); expect( !!assertions[assertionName], `Assertion with name: ${assertionName} not found!`, ).to.be.equal(true); - const { evmOperationalWalletPublicKey, evmOperationalWalletPrivateKey } = - this.state.nodes[node - 1].configuration.modules.blockchain.implementation.hardhat - .config; + const assertion = assertions[assertionName]; const result = await this.state.nodes[node - 1].client - .publish(assertion, { evmOperationalWalletPublicKey, evmOperationalWalletPrivateKey }) + .publish(assertion) .catch((error) => { assert.fail(`Error while trying to publish assertion. ${error}`); }); const { operationId } = result.operation; - this.state.lastPublishData = { + this.state.latestPublishData = { nodeId: node - 1, UAL: result.UAL, assertionId: result.assertionId, @@ -40,11 +38,12 @@ When( }; }, ); + When( - /^I call publish on ot-node (\d+) directly with ([^"]*)/, + /^I call Publish directly on the node (\d+) with ([^"]*)/, { timeout: 70000 }, async function publish(node, requestName) { - this.logger.log(`I call publish on ot-node ${node} directly`); + this.logger.log(`I call publish on the node ${node} directly`); expect( !!requests[requestName], `Request body with name: ${requestName} not found!`, @@ -55,63 +54,65 @@ When( requestBody, ); const { operationId } = result.data; - this.state.lastPublishData = { + this.state.latestPublishData = { nodeId: node - 1, operationId, }; }, ); -Given('I wait for last publish to finalize', { timeout: 80000 }, async function publishFinalize() { - this.logger.log('I wait for last publish to finalize'); +When('I wait for latest Publish to finalize', { timeout: 80000 }, async function publishFinalize() { + this.logger.log('I wait for latest publish to finalize'); expect( - !!this.state.lastPublishData, - 'Last publish data is undefined. Publish is not started.', + !!this.state.latestPublishData, + 'Latest publish data is undefined. Publish was not started.', ).to.be.equal(true); - const publishData = this.state.lastPublishData; + const publishData = this.state.latestPublishData; let retryCount = 0; const maxRetryCount = 5; for (retryCount = 0; retryCount < maxRetryCount; retryCount += 1) { this.logger.log( - `Getting publish result for operation id: ${publishData.operationId} on node: ${publishData.nodeId}`, + `Getting publish result for operation id: ${publishData.operationId} on the node: ${publishData.nodeId}`, ); // eslint-disable-next-line no-await-in-loop const publishResult = await httpApiHelper.getOperationResult( this.state.nodes[publishData.nodeId].nodeRpcUrl, + 'publish', publishData.operationId, ); this.logger.log(`Operation status: ${publishResult.data.status}`); if (['COMPLETED', 'FAILED'].includes(publishResult.data.status)) { - this.state.lastPublishData.result = publishResult; - this.state.lastPublishData.status = publishResult.data.status; - this.state.lastPublishData.errorType = publishResult.data.data?.errorType; + this.state.latestPublishData.result = publishResult; + this.state.latestPublishData.status = publishResult.data.status; + this.state.latestPublishData.errorType = publishResult.data.data?.errorType; break; } if (retryCount === maxRetryCount - 1) { - assert.fail('Unable to get publish result'); + assert.fail('Unable to fetch publish result'); } // eslint-disable-next-line no-await-in-loop await setTimeout(4000); } }); -Given( +When( /I wait for (\d+) seconds and check operation status/, { timeout: 120000 }, async function publishWait(numberOfSeconds) { this.logger.log(`I wait for ${numberOfSeconds} seconds`); expect( - !!this.state.lastPublishData, - 'Last publish data is undefined. Publish is not started.', + !!this.state.latestPublishData, + 'Latest publish data is undefined. Publish is not started.', ).to.be.equal(true); - const publishData = this.state.lastPublishData; + const publishData = this.state.latestPublishData; this.logger.log( - `Getting publish result for operation id: ${publishData.operationId} on node: ${publishData.nodeId}`, + `Getting publish result for operation id: ${publishData.operationId} on the node: ${publishData.nodeId}`, ); await setTimeout(numberOfSeconds * 1000); // eslint-disable-next-line no-await-in-loop - this.state.lastPublishData.result = await httpApiHelper.getOperationResult( + this.state.latestPublishData.result = await httpApiHelper.getOperationResult( this.state.nodes[publishData.nodeId].nodeRpcUrl, + 'publish', publishData.operationId, ); }, diff --git a/test/bdd/steps/api/resolve.mjs b/test/bdd/steps/api/resolve.mjs index eaa7d7cd9..695715b31 100644 --- a/test/bdd/steps/api/resolve.mjs +++ b/test/bdd/steps/api/resolve.mjs @@ -1,32 +1,29 @@ -import { When, Given } from '@cucumber/cucumber'; +import { When } from '@cucumber/cucumber'; import { expect, assert } from 'chai'; import { setTimeout } from 'timers/promises'; import HttpApiHelper from "../../../utilities/http-api-helper.mjs"; -import {readFile} from "fs/promises"; -const requests = JSON.parse(await readFile("test/bdd/steps/api/datasets/requests.json")); const httpApiHelper = new HttpApiHelper() - When( - /^I get operation result from node (\d+) for last published assertion/, + /^I get operation result from node (\d+) for latest published assertion/, { timeout: 120000 }, async function resolveCall(node) { - this.logger.log('I call get result for the last operation'); + this.logger.log('I call get result for the latest operation'); expect( - !!this.state.lastPublishData, - 'Last publish data is undefined. Publish is not finalized.', + !!this.state.latestPublishData, + 'Latest publish data is undefined. Publish is not finalized.', ).to.be.equal(true); try { const result = await this.state.nodes[node - 1].client - .getResult(this.state.lastPublishData.UAL) + .get(this.state.latestPublishData.UAL) .catch((error) => { assert.fail(`Error while trying to resolve assertion. ${error}`); }); const { operationId } = result.operation; - this.state.lastGetData = { + this.state.latestGetData = { nodeId: node - 1, operationId, result, @@ -39,32 +36,33 @@ When( }, ); -Given( - 'I wait for last resolve to finalize', +When( + 'I wait for latest resolve to finalize', { timeout: 120000 }, async function resolveFinalizeCall() { - this.logger.log('I wait for last resolve to finalize'); + this.logger.log('I wait for latest resolve to finalize'); expect( - !!this.state.lastGetData, - 'Last resolve data is undefined. Resolve is not started.', + !!this.state.latestGetData, + 'Latest resolve data is undefined. Resolve is not started.', ).to.be.equal(true); - const resolveData = this.state.lastGetData; + const resolveData = this.state.latestGetData; let retryCount = 0; const maxRetryCount = 5; for (retryCount = 0; retryCount < maxRetryCount; retryCount += 1) { this.logger.log( - `Getting resolve result for operation id: ${resolveData.operationId} on node: ${resolveData.nodeId}`, + `Getting resolve result for operation id: ${resolveData.operationId} on the node: ${resolveData.nodeId}`, ); // eslint-disable-next-line no-await-in-loop const resolveResult = await httpApiHelper.getOperationResult( this.state.nodes[resolveData.nodeId].nodeRpcUrl, + 'get', resolveData.operationId, ); this.logger.log(`Operation status: ${resolveResult.data.status}`); if (['COMPLETED', 'FAILED'].includes(resolveResult.data.status)) { - this.state.lastGetData.result = resolveResult; - this.state.lastGetData.status = resolveResult.data.status; - this.state.lastGetData.errorType = resolveResult.data.data?.errorType; + this.state.latestGetData.result = resolveResult; + this.state.latestGetData.status = resolveResult.data.status; + this.state.latestGetData.errorType = resolveResult.data.data?.errorType; break; } if (retryCount === maxRetryCount - 1) { @@ -76,17 +74,17 @@ Given( }, ); -Given(/Last resolve returned valid result$/, { timeout: 120000 }, async function resolveCall() { - this.logger.log('Last resolve returned valid result'); +When(/Latest resolve returned valid result$/, { timeout: 120000 }, async function resolveCall() { + this.logger.log('Latest resolve returned valid result'); expect( - !!this.state.lastGetData, - 'Last resolve data is undefined. Resolve is not started.', + !!this.state.latestGetData, + 'Latest resolve data is undefined. Resolve is not started.', ).to.be.equal(true); expect( - !!this.state.lastGetData.result, - 'Last publish data result is undefined. Publish is not finished.', + !!this.state.latestGetData.result, + 'Latest publish data result is undefined. Publish is not finished.', ).to.be.equal(true); - const resolveData = this.state.lastGetData; + const resolveData = this.state.latestGetData; expect( Array.isArray(resolveData.result.data), 'Resolve result data expected to be array', @@ -95,30 +93,7 @@ Given(/Last resolve returned valid result$/, { timeout: 120000 }, async function // expect(resolveData.result.data.length, 'Returned data array length').to.be.equal(1); // const resolvedAssertion = resolveData.result.data[0].assertion.data; - // const publishedAssertion = this.state.lastPublishData.assertion; + // const publishedAssertion = this.state.latestPublishData.assertion; // assert.equal(sortedStringify(publishedAssertion), sortedStringify(resolvedAssertion)); }); -Given( - /^I call get directly to ot-node (\d+) with ([^"]*)/, - { timeout: 30000 }, - async function getFromNode(node, requestName) { - this.logger.log(`I call get on ot-node ${node} directly`); - if (requestName !== 'lastPublishedAssetUAL') { - expect( - !!requests[requestName], - `Request body with name: ${requestName} not found!`, - ).to.be.equal(true); - } - const requestBody = - requestName !== 'lastPublishedAssetUAL' - ? requests[requestName] - : { id: this.state.lastPublishData.UAL }; - const result = await httpApiHelper.get(this.state.nodes[node - 1].nodeRpcUrl, requestBody); - const { operationId } = result.data; - this.state.lastGetData = { - nodeId: node - 1, - operationId, - }; - }, -); diff --git a/test/bdd/steps/api/update.mjs b/test/bdd/steps/api/update.mjs index 7a3f9a84b..ebf4c4651 100644 --- a/test/bdd/steps/api/update.mjs +++ b/test/bdd/steps/api/update.mjs @@ -1,4 +1,4 @@ -import { When, Given } from '@cucumber/cucumber'; +import { When } from '@cucumber/cucumber'; import { expect, assert } from 'chai'; import { setTimeout } from 'timers/promises'; import { readFile } from 'fs/promises'; @@ -10,26 +10,24 @@ const requests = JSON.parse(await readFile('test/bdd/steps/api/datasets/requests const httpApiHelper = new HttpApiHelper(); When( - /^I call update on node (\d+) for last publish UAL with ([^"]*)/, + /^I call Update on the node (\d+) for the latest published UAL with ([^"]*)/, { timeout: 120000 }, async function update(node, assertionName) { - this.logger.log(`I call update route on node ${node}`); + this.logger.log(`I call update route on the node ${node}`); expect( !!assertions[assertionName], `Assertion with name: ${assertionName} not found!`, ).to.be.equal(true); - const { evmOperationalWalletPublicKey, evmOperationalWalletPrivateKey } = - this.state.nodes[node - 1].configuration.modules.blockchain.implementation.hardhat - .config; + const assertion = assertions[assertionName]; - const { UAL } = this.state.lastPublishData; + const { UAL } = this.state.latestPublishData; const result = await this.state.nodes[node - 1].client - .update(UAL, assertion, { evmOperationalWalletPublicKey, evmOperationalWalletPrivateKey }) + .update(UAL, assertion) .catch((error) => { assert.fail(`Error while trying to update assertion. ${error}`); }); const { operationId } = result.operation; - this.state.lastUpdateData = { + this.state.latestUpdateData = { nodeId: node - 1, UAL, assertionId: result.assertionId, @@ -41,11 +39,12 @@ When( }; }, ); + When( - /^I call update on ot-node (\d+) directly with ([^"]*)/, + /^I call Update directly on the node (\d+) with ([^"]*)/, { timeout: 70000 }, async function publish(node, requestName) { - this.logger.log(`I call update on ot-node ${node} directly`); + this.logger.log(`I call update on the node ${node} directly`); expect( !!requests[requestName], `Request body with name: ${requestName} not found!`, @@ -56,40 +55,41 @@ When( requestBody, ); const { operationId } = result.data; - this.state.lastUpdateData = { + this.state.latestUpdateData = { nodeId: node - 1, operationId, }; }, ); -Given('I wait for last update to finalize', { timeout: 80000 }, async function publishFinalize() { - this.logger.log('I wait for last update to finalize'); +When('I wait for latest Update to finalize', { timeout: 80000 }, async function publishFinalize() { + this.logger.log('I wait for latest update to finalize'); expect( - !!this.state.lastUpdateData, - 'Last update data is undefined. Update is not started.', + !!this.state.latestUpdateData, + 'Latest update data is undefined. Update was not started.', ).to.be.equal(true); - const updateData = this.state.lastUpdateData; + const updateData = this.state.latestUpdateData; let retryCount = 0; const maxRetryCount = 5; for (retryCount = 0; retryCount < maxRetryCount; retryCount += 1) { this.logger.log( - `Getting Update result for operation id: ${updateData.operationId} on node: ${updateData.nodeId}`, + `Getting Update result for operation id: ${updateData.operationId} on the node: ${updateData.nodeId}`, ); // eslint-disable-next-line no-await-in-loop const updateResult = await httpApiHelper.getOperationResult( this.state.nodes[updateData.nodeId].nodeRpcUrl, + 'update', updateData.operationId, ); this.logger.log(`Operation status: ${updateResult.data.status}`); if (['COMPLETED', 'FAILED'].includes(updateResult.data.status)) { - this.state.lastUpdateData.result = updateResult; - this.state.lastUpdateData.status = updateResult.data.status; - this.state.lastUpdateData.errorType = updateResult.data.data?.errorType; + this.state.latestUpdateData.result = updateResult; + this.state.latestUpdateData.status = updateResult.data.status; + this.state.latestUpdateData.errorType = updateResult.data.data?.errorType; break; } if (retryCount === maxRetryCount - 1) { - assert.fail('Unable to get update result'); + assert.fail('Unable to fetch update result'); } // eslint-disable-next-line no-await-in-loop await setTimeout(4000); diff --git a/test/bdd/steps/common.mjs b/test/bdd/steps/common.mjs index c2a3c5138..5a09f8857 100644 --- a/test/bdd/steps/common.mjs +++ b/test/bdd/steps/common.mjs @@ -1,10 +1,11 @@ -import { Given } from '@cucumber/cucumber'; +import { Given, Then } from '@cucumber/cucumber'; import { expect, assert } from 'chai'; import fs from 'fs'; import { setTimeout as sleep } from 'timers/promises'; import DkgClientHelper from '../../utilities/dkg-client-helper.mjs'; import StepsUtils from '../../utilities/steps-utils.mjs'; +import FileService from "../../../src/service/file-service.js"; const stepsUtils = new StepsUtils(); @@ -53,15 +54,21 @@ Given( const client = new DkgClientHelper({ endpoint: 'http://localhost', port: rpcPort, - useSSL: false, - timeout: 25, - loglevel: 'trace', + blockchain: { + name: 'hardhat', + publicKey: wallet.address, + privateKey: wallet.privateKey, + }, + maxNumberOfRetries: 5, + frequency: 2, + contentType: 'all', }); this.state.nodes[nodeIndex] = { client, forkedNode, configuration: nodeConfiguration, nodeRpcUrl: `http://localhost:${rpcPort}`, + fileService: new FileService({config: nodeConfiguration, logger: this.logger}), }; } nodesStarted += 1; @@ -125,6 +132,7 @@ Given( forkedNode, configuration: nodeConfiguration, nodeRpcUrl: `http://localhost:${rpcPort}`, + fileService: new FileService({config: nodeConfiguration, logger: this.logger}), }); } done(); @@ -162,9 +170,8 @@ Given( Object.prototype.hasOwnProperty.call(nodeConfiguration, propertyNameSplit[0]), `Property ${propertyName} doesn't exist`, ).to.be.equal(true); - const propertyNameSplitLen = propertyNameSplit.length; let propName = nodeConfiguration; - for (let i = 0; i < propertyNameSplitLen - 1; i += 1) { + for (let i = 0; i < propertyNameSplit.length - 1; i += 1) { propName = propName[propertyNameSplit[i]]; } if (propName[propertyNameSplit.slice(-1)] !== undefined) { @@ -189,41 +196,41 @@ Given( const client = new DkgClientHelper({ endpoint: 'http://localhost', port: rpcPort, - useSSL: false, - timeout: 25, - loglevel: 'trace', + blockchain: { + name: 'hardhat', + publicKey: wallet.address, + privateKey: wallet.privateKey, + }, + maxNumberOfRetries: 5, + frequency: 2, + contentType: 'all', }); this.state.nodes[nodeIndex] = { client, - clientConfig: { - endpoint: 'http://localhost', - port: rpcPort, - useSSL: false, - timeout: 25, - loglevel: 'trace', - }, forkedNode, configuration: nodeConfiguration, nodeRpcUrl: `http://localhost:${rpcPort}`, + fileService: new FileService({config: nodeConfiguration, logger: this.logger}), }; } done(); }); }, ); -Given( - /Last (Get|Publish|Update) operation finished with status: ([COMPLETED|FAILED|PublishValidateAssertionError|PublishStartError|GetAssertionIdError|GetNetworkError|GetLocalError|PublishRouteError]+)$/, + +Then( + /Latest (Get|Publish|Update) operation finished with status: ([COMPLETED|FAILED|PublishValidateAssertionError|PublishStartError|GetAssertionIdError|GetNetworkError|GetLocalError|PublishRouteError]+)$/, { timeout: 120000 }, - async function lastResolveFinishedCall(operationName, status) { - this.logger.log(`Last ${operationName} operation finished with status: ${status}`); - const operationData = `last${operationName}Data`; + async function latestResolveFinishedCall(operationName, status) { + this.logger.log(`Latest ${operationName} operation finished with status: ${status}`); + const operationData = `latest${operationName}Data`; expect( !!this.state[operationData], - `Last ${operationName} result is undefined. ${operationData} result not started.`, + `Latest ${operationName} result is undefined. ${operationData} result not started.`, ).to.be.equal(true); expect( !!this.state[operationData].result, - `Last ${operationName} result data result is undefined. ${operationData} result is not finished.`, + `Latest ${operationName} result data result is undefined. ${operationData} result is not finished.`, ).to.be.equal(true); expect( @@ -247,3 +254,8 @@ Given(/^I set R0 to be (\d+)$/, { timeout: 100000 }, async function waitFor(r0) this.logger.log(`I set R0 to be ${r0}`); await this.state.localBlockchain.setR0(r0); }); + +Given(/^I set finalizationCommitsNumber to be (\d+)$/, { timeout: 100000 }, async function waitFor(finalizationCommitsNumber) { + this.logger.log(`I set finalizationCommitsNumber to be ${finalizationCommitsNumber}`); + await this.state.localBlockchain.setFinalizationCommitsNumber(finalizationCommitsNumber); +}); diff --git a/test/bdd/steps/hooks.mjs b/test/bdd/steps/hooks.mjs index 27c9fc985..9fbb4391c 100644 --- a/test/bdd/steps/hooks.mjs +++ b/test/bdd/steps/hooks.mjs @@ -2,9 +2,9 @@ import 'dotenv/config'; import { Before, BeforeAll, After, AfterAll } from '@cucumber/cucumber'; import slugify from 'slugify'; import fs from 'fs'; +import mysql from "mysql2"; import { NODE_ENVIRONMENTS } from '../../../src/constants/constants.js'; import TripleStoreModuleManager from "../../../src/modules/triple-store/triple-store-module-manager.js"; -import mysql from "mysql2"; process.env.NODE_ENV = NODE_ENVIRONMENTS.TEST; @@ -29,22 +29,28 @@ Before(function beforeMethod(testCase, done) { After(function afterMethod(testCase, done) { const tripleStoreConfiguration = []; const databaseNames = []; + const promises = []; for (const key in this.state.nodes) { this.state.nodes[key].forkedNode.kill(); tripleStoreConfiguration.push({modules: {tripleStore: this.state.nodes[key].configuration.modules.tripleStore}}); databaseNames.push(this.state.nodes[key].configuration.operationalDatabase.databaseName); + const dataFolderPath = this.state.nodes[key].fileService.getDataFolderPath(); + promises.push(this.state.nodes[key].fileService.removeFolder(dataFolderPath)); } this.state.bootstraps.forEach((node) => { node.forkedNode.kill(); tripleStoreConfiguration.push({modules: {tripleStore: node.configuration.modules.tripleStore}}); databaseNames.push(node.configuration.operationalDatabase.databaseName); + const dataFolderPath = node.fileService.getDataFolderPath(); + promises.push(node.fileService.removeFolder(dataFolderPath)); }); if (this.state.localBlockchain) { + this.logger.info('Stopping local blockchain!'); this.state.localBlockchain.stop(); + this.state.localBlockchain = null; } this.logger.log('After test hook, cleaning repositories'); - const promises = []; const con = mysql.createConnection({ host: 'localhost', user: 'root', @@ -58,18 +64,17 @@ After(function afterMethod(testCase, done) { tripleStoreConfiguration.forEach((config) => { promises.push(async () => { const tripleStoreModuleManager = new TripleStoreModuleManager({config, logger: this.logger}); - await tripleStoreModuleManager.initialize() + await tripleStoreModuleManager.initialize(); for (const implementationName of tripleStoreModuleManager.getImplementationNames()) { - const {config} = tripleStoreModuleManager.getImplementation(implementationName); - Object.keys(config.repositories).map(async (repository) => { - console.log('Removing triple store configuration:', JSON.stringify(config, null, 4)); + const {tripleStoreConfig} = tripleStoreModuleManager.getImplementation(implementationName); + Object.keys(tripleStoreConfig.repositories).map(async (repository) => { + this.logger.log('Removing triple store configuration:', JSON.stringify(tripleStoreConfig, null, 4)); await tripleStoreModuleManager.deleteRepository(implementationName, repository); } ) } }) }) - Promise.all(promises) .then(() => { con.end(); diff --git a/test/bdd/steps/lib/local-blockchain.mjs b/test/bdd/steps/lib/local-blockchain.mjs index aeb799306..65515e49e 100644 --- a/test/bdd/steps/lib/local-blockchain.mjs +++ b/test/bdd/steps/lib/local-blockchain.mjs @@ -115,18 +115,27 @@ class LocalBlockchain { } } - async setR1(R1) { - console.log(`Setting R1 in parameters storage to: ${R1}`); - const encodedData = this.ParametersStorageInterface.encodeFunctionData('setR1', [R1]); + async setR0(r0) { + console.log(`Setting R0 in parameters storage to: ${r0}`); + const encodedData = this.ParametersStorageInterface.encodeFunctionData('setR0', [r0]); const parametersStorageAddress = await this.hubContract.getContractAddress( 'ParametersStorage', ); await this.HubControllerContract.forwardCall(parametersStorageAddress, encodedData); } - async setR0(R0) { - console.log(`Setting R0 in parameters storage to: ${R0}`); - const encodedData = this.ParametersStorageInterface.encodeFunctionData('setR0', [R0]); + async setR1(r1) { + console.log(`Setting R1 in parameters storage to: ${r1}`); + const encodedData = this.ParametersStorageInterface.encodeFunctionData('setR1', [r1]); + const parametersStorageAddress = await this.hubContract.getContractAddress( + 'ParametersStorage', + ); + await this.HubControllerContract.forwardCall(parametersStorageAddress, encodedData); + } + + async setFinalizationCommitsNumber(commitsNumber) { + console.log(`Setting finalizationCommitsNumber in parameters storage to: ${commitsNumber}`); + const encodedData = this.ParametersStorageInterface.encodeFunctionData('setFinalizationCommitsNumber', [commitsNumber]); const parametersStorageAddress = await this.hubContract.getContractAddress( 'ParametersStorage', ); diff --git a/test/bdd/steps/lib/state.mjs b/test/bdd/steps/lib/state.mjs index 2fcc82bc0..727635eeb 100644 --- a/test/bdd/steps/lib/state.mjs +++ b/test/bdd/steps/lib/state.mjs @@ -12,31 +12,38 @@ const state = { 0: { client: {}, fork: {}, + fileService: {}, + configuration: {}, + nodeRpcUrl: '' }, 1: { client: {}, fork: {}, + fileService: {}, + configuration: {}, + nodeRpcUrl: '' }, }, bootstraps: [], - lastPublishData: { + latestPublishData: { nodeId: 1, operationId: '', keywords: ['', ''], assertion: {}, result: {}, }, - lastGetData: { + latestGetData: { nodeId: 1, operationId: '', assertionIds: ['', ''], result: {}, }, - lastUpdateData: { + latestUpdateData: { nodeId: 1, operationId: '', assertionIds: ['', ''], result: {}, }, + latestError: {}, scenarionLogDir: '', }; diff --git a/test/utilities/dkg-client-helper.mjs b/test/utilities/dkg-client-helper.mjs index c9c7e172f..bc18895fc 100644 --- a/test/utilities/dkg-client-helper.mjs +++ b/test/utilities/dkg-client-helper.mjs @@ -10,52 +10,35 @@ class DkgClientHelper { return this.client.node.info(); } - async publish(data, wallet) { + async publish(data) { const options = { visibility: 'public', epochsNum: 5, - maxNumberOfRetries: 5, hashFunctionId: CONTENT_ASSET_HASH_FUNCTION_ID, - blockchain: { - name: 'hardhat', - publicKey: wallet.evmOperationalWalletPublicKey, - privateKey: wallet.evmOperationalWalletPrivateKey, - }, }; + return this.client.asset.create(data, options); } - async update(ual, assertion, wallet) { + async update(ual, assertion) { const options = { - maxNumberOfRetries: 5, - blockchain: { - name: 'hardhat', - publicKey: wallet.evmOperationalWalletPublicKey, - privateKey: wallet.evmOperationalWalletPrivateKey, - }, + hashFunctionId: CONTENT_ASSET_HASH_FUNCTION_ID, }; + return this.client.asset.update(ual, assertion, options); } - async get(ids) { - return this.client._getRequest({ - ids, - }); - } + async get(ual, state) { + const options = { + state, + validate: true, + }; - async query(query) { - return this.client._queryRequest({ - query, - }); + return this.client.asset.get(ual, options); } - async getResult(UAL) { - const getOptions = { - validate: true, - commitOffset: 0, - maxNumberOfRetries: 5, - }; - return this.client.asset.get(UAL, getOptions).catch(() => {}); + async query(query) { + return this.client.query(query); } } diff --git a/test/utilities/http-api-helper.mjs b/test/utilities/http-api-helper.mjs index aabcc7192..bac9c33d9 100644 --- a/test/utilities/http-api-helper.mjs +++ b/test/utilities/http-api-helper.mjs @@ -2,69 +2,36 @@ import axios from 'axios'; class HttpApiHelper { async info(nodeRpcUrl) { - try { - const response = await axios({ - method: 'get', - url: `${nodeRpcUrl}/info`, - }); - - return response; - } catch (e) { - throw Error(`Unable to get info: ${e.message}`); - } + return this._sendRequest('get', `${nodeRpcUrl}/info`); } - async get(nodeRpcUrl, ual) { - // Not sure if header is needed - try { - const response = await axios({ - method: 'post', - url: `${nodeRpcUrl}/get`, - data: ual, - headers: { - 'Content-Type': 'application/json', - }, - }); - - return response; - } catch (e) { - throw Error(`Unable to GET: ${e.message}`); - } + async get(nodeRpcUrl, requestBody) { + return this._sendRequest('post', `${nodeRpcUrl}/get`, requestBody); } - async getOperationResult(nodeRpcUrl, operationId) { - try { - const response = await axios({ - method: 'get', - url: `${nodeRpcUrl}/publish/${operationId}`, - }); - - return response; - } catch (e) { - throw Error(`Unable to PUBLISH: ${e.message}`); - } + async getOperationResult(nodeRpcUrl, operationName, operationId) { + return this._sendRequest('get', `${nodeRpcUrl}/${operationName}/${operationId}`); } async publish(nodeRpcUrl, requestBody) { - try { - const response = await axios({ - method: 'post', - url: `${nodeRpcUrl}/publish`, - data: requestBody, - }); - - return response; - } catch (e) { - throw Error(`Unable to publish: ${e.message}`); - } + return this._sendRequest('post', `${nodeRpcUrl}/publish`, requestBody); } + async update(nodeRpcUrl, requestBody) { + return this._sendRequest('post', `${nodeRpcUrl}/update`, requestBody); + } + + async _sendRequest(method, url, data) { return axios({ - method: 'post', - url: `${nodeRpcUrl}/update`, - data: requestBody, - }).catch((e) => { - throw Error(`Unable to update: ${e.message}`); + method, + url, + ...data && { data }, + }).catch((error) => { + const errorWithStatus = new Error(error.message); + if (error.response) { + errorWithStatus.statusCode = error.response.status; + } + throw errorWithStatus; }); } } diff --git a/test/utilities/steps-utils.mjs b/test/utilities/steps-utils.mjs index cf79f4e1a..e17a5eb64 100644 --- a/test/utilities/steps-utils.mjs +++ b/test/utilities/steps-utils.mjs @@ -124,10 +124,6 @@ class StepsUtils { graphDatabase: { name: nodeName, }, - minimumAckResponses: { - publish: 2, - get: 1, - }, }; } } From 0c01213335ff3e5ea3095b59a4c38106641cf31a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 20 Jun 2023 09:51:16 +0200 Subject: [PATCH 17/33] Bump @openzeppelin/contracts from 4.9.1 to 4.9.2 (#2590) Bumps [@openzeppelin/contracts](https://github.com/OpenZeppelin/openzeppelin-contracts) from 4.9.1 to 4.9.2. - [Release notes](https://github.com/OpenZeppelin/openzeppelin-contracts/releases) - [Changelog](https://github.com/OpenZeppelin/openzeppelin-contracts/blob/v4.9.2/CHANGELOG.md) - [Commits](https://github.com/OpenZeppelin/openzeppelin-contracts/compare/v4.9.1...v4.9.2) --- updated-dependencies: - dependency-name: "@openzeppelin/contracts" dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- package-lock.json | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/package-lock.json b/package-lock.json index 90f9414b5..7ad232345 100644 --- a/package-lock.json +++ b/package-lock.json @@ -4324,9 +4324,9 @@ } }, "node_modules/@openzeppelin/contracts": { - "version": "4.9.1", - "resolved": "https://registry.npmjs.org/@openzeppelin/contracts/-/contracts-4.9.1.tgz", - "integrity": "sha512-aLDTLu/If1qYIFW5g4ZibuQaUsFGWQPBq1mZKp/txaebUnGHDmmiBhRLY1tDNedN0m+fJtKZ1zAODS9Yk+V6uA==" + "version": "4.9.2", + "resolved": "https://registry.npmjs.org/@openzeppelin/contracts/-/contracts-4.9.2.tgz", + "integrity": "sha512-mO+y6JaqXjWeMh9glYVzVu8HYPGknAAnWyxTRhGeckOruyXQMNnlcW6w/Dx9ftLeIQk6N+ZJFuVmTwF7lEIFrg==" }, "node_modules/@polkadot/api": { "version": "9.14.2", @@ -22942,9 +22942,9 @@ "optional": true }, "@openzeppelin/contracts": { - "version": "4.9.1", - "resolved": "https://registry.npmjs.org/@openzeppelin/contracts/-/contracts-4.9.1.tgz", - "integrity": "sha512-aLDTLu/If1qYIFW5g4ZibuQaUsFGWQPBq1mZKp/txaebUnGHDmmiBhRLY1tDNedN0m+fJtKZ1zAODS9Yk+V6uA==" + "version": "4.9.2", + "resolved": "https://registry.npmjs.org/@openzeppelin/contracts/-/contracts-4.9.2.tgz", + "integrity": "sha512-mO+y6JaqXjWeMh9glYVzVu8HYPGknAAnWyxTRhGeckOruyXQMNnlcW6w/Dx9ftLeIQk6N+ZJFuVmTwF7lEIFrg==" }, "@polkadot/api": { "version": "9.14.2", From 5c7273da2c2f9264fd21e51eeddf2c35d5310d39 Mon Sep 17 00:00:00 2001 From: zeroxbt <89495162+zeroxbt@users.noreply.github.com> Date: Tue, 20 Jun 2023 09:51:47 +0200 Subject: [PATCH 18/33] fix remove cached folder from pending storage (#2589) --- src/service/pending-storage-service.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/service/pending-storage-service.js b/src/service/pending-storage-service.js index 18d1a94e1..a882ce40a 100644 --- a/src/service/pending-storage-service.js +++ b/src/service/pending-storage-service.js @@ -67,6 +67,7 @@ class PendingStorageService { repository, blockchain, contract, + tokenId, ); await this.fileService.removeFolder(pendingStorageFolderPath); } From 54c53d128a6bbaa0aa192d774495d3be5cf29662 Mon Sep 17 00:00:00 2001 From: djordjekovac Date: Tue, 20 Jun 2023 11:38:21 +0200 Subject: [PATCH 19/33] Update release-drafter-config.yml --- .github/workflows/release-drafter-config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release-drafter-config.yml b/.github/workflows/release-drafter-config.yml index e48dfad82..069a2ed64 100644 --- a/.github/workflows/release-drafter-config.yml +++ b/.github/workflows/release-drafter-config.yml @@ -3,7 +3,7 @@ name: release-drafter on: push: branches: - - v6/develop + - develop jobs: update_release_draft: From 1287f416dacf3d41dfa67fed31b3cc5bd5ea470d Mon Sep 17 00:00:00 2001 From: zeroxbt <89495162+zeroxbt@users.noreply.github.com> Date: Tue, 20 Jun 2023 12:05:07 +0200 Subject: [PATCH 20/33] bump version (#2591) --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 7ad232345..f3230bba1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "origintrail_node", - "version": "6.0.11", + "version": "6.0.11+hotfix.1", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "origintrail_node", - "version": "6.0.11", + "version": "6.0.11+hotfix.1", "license": "ISC", "dependencies": { "@comunica/query-sparql": "^2.4.3", diff --git a/package.json b/package.json index aaeca8c03..abb4900c7 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "origintrail_node", - "version": "6.0.11", + "version": "6.0.11+hotfix.1", "description": "OTNode V6", "main": "index.js", "type": "module", From 3be87d0322509b4f0edf80849e462f3e2930ab31 Mon Sep 17 00:00:00 2001 From: zeroxbt <89495162+zeroxbt@users.noreply.github.com> Date: Tue, 20 Jun 2023 15:40:17 +0200 Subject: [PATCH 21/33] fix undefined public assertion on operation completed (#2594) * fix undefined public assertion on operation completed * fix unit tests --- .../update/sender/v1.0.0/v1-0-0-update-request-command.js | 7 +++---- src/service/operation-service.js | 4 +++- src/service/publish-service.js | 2 +- src/service/update-service.js | 2 +- test/unit/service/publish-service.test.js | 2 +- test/unit/service/update-service.test.js | 2 +- 6 files changed, 10 insertions(+), 9 deletions(-) diff --git a/src/commands/protocols/update/sender/v1.0.0/v1-0-0-update-request-command.js b/src/commands/protocols/update/sender/v1.0.0/v1-0-0-update-request-command.js index eb89dd39d..cda7aec63 100644 --- a/src/commands/protocols/update/sender/v1.0.0/v1-0-0-update-request-command.js +++ b/src/commands/protocols/update/sender/v1.0.0/v1-0-0-update-request-command.js @@ -10,10 +10,9 @@ class UpdateRequestCommand extends ProtocolRequestCommand { } async prepareMessage(command) { - const data = await this.operationIdService.getCachedOperationIdData( - command.data.operationId, - ); - const { assertion } = data.public; + const { + public: { assertion }, + } = await this.operationIdService.getCachedOperationIdData(command.data.operationId); return { assertion, diff --git a/src/service/operation-service.js b/src/service/operation-service.js index 8f9748305..13f4d6428 100644 --- a/src/service/operation-service.js +++ b/src/service/operation-service.js @@ -68,7 +68,9 @@ class OperationService { OPERATION_STATUS.COMPLETED, ); - await this.operationIdService.cacheOperationIdData(operationId, responseData); + if (responseData != null) { + await this.operationIdService.cacheOperationIdData(operationId, responseData); + } for (const status of endStatuses) { // eslint-disable-next-line no-await-in-loop diff --git a/src/service/publish-service.js b/src/service/publish-service.js index 96d2feb40..12b16edbf 100644 --- a/src/service/publish-service.js +++ b/src/service/publish-service.js @@ -71,7 +71,7 @@ class PublishService extends OperationService { } } if (allCompleted) { - await this.markOperationAsCompleted(operationId, {}, this.completedStatuses); + await this.markOperationAsCompleted(operationId, null, this.completedStatuses); this.logResponsesSummary(completedNumber, failedNumber); this.logger.info( `${this.operationName} with operation id: ${operationId} with status: ${ diff --git a/src/service/update-service.js b/src/service/update-service.js index bc8ee70d3..758e4de96 100644 --- a/src/service/update-service.js +++ b/src/service/update-service.js @@ -71,7 +71,7 @@ class UpdateService extends OperationService { } } if (allCompleted) { - await this.markOperationAsCompleted(operationId, {}, this.completedStatuses); + await this.markOperationAsCompleted(operationId, null, this.completedStatuses); this.logResponsesSummary(completedNumber, failedNumber); this.logger.info( `${this.operationName} with operation id: ${operationId} with status: ${ diff --git a/test/unit/service/publish-service.test.js b/test/unit/service/publish-service.test.js index 07a4d6052..fad7dc6ad 100644 --- a/test/unit/service/publish-service.test.js +++ b/test/unit/service/publish-service.test.js @@ -59,7 +59,7 @@ describe('Publish service test', async () => { const returnedResponses = publishService.repositoryModuleManager.getAllResponseStatuses(); expect(cacheOperationIdDataSpy.calledWith('5195d01a-b437-4aae-b388-a77b9fa715f1', {})).to.be - .true; + .false; expect(returnedResponses.length).to.be.equal(2); diff --git a/test/unit/service/update-service.test.js b/test/unit/service/update-service.test.js index 4e524cde3..46409a856 100644 --- a/test/unit/service/update-service.test.js +++ b/test/unit/service/update-service.test.js @@ -62,7 +62,7 @@ describe('Update service test', async () => { expect(returnedResponses.length).to.be.equal(2); expect(cacheOperationIdDataSpy.calledWith('5195d01a-b437-4aae-b388-a77b9fa715f1', {})).to.be - .true; + .false; expect( returnedResponses[returnedResponses.length - 1].status === From 3c6fec7989bd623db667443b8a5e6f66c51dacf3 Mon Sep 17 00:00:00 2001 From: zeroxbt <89495162+zeroxbt@users.noreply.github.com> Date: Tue, 20 Jun 2023 15:43:25 +0200 Subject: [PATCH 22/33] bump version (#2595) * fix undefined public assertion on operation completed * fix unit tests * bump version to 6.0.11 --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index f3230bba1..7ad232345 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "origintrail_node", - "version": "6.0.11+hotfix.1", + "version": "6.0.11", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "origintrail_node", - "version": "6.0.11+hotfix.1", + "version": "6.0.11", "license": "ISC", "dependencies": { "@comunica/query-sparql": "^2.4.3", diff --git a/package.json b/package.json index abb4900c7..aaeca8c03 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "origintrail_node", - "version": "6.0.11+hotfix.1", + "version": "6.0.11", "description": "OTNode V6", "main": "index.js", "type": "module", From 92b7e1111f7eec34d365380d18ba7f42996b474a Mon Sep 17 00:00:00 2001 From: zeroxbt <89495162+zeroxbt@users.noreply.github.com> Date: Tue, 20 Jun 2023 17:07:41 +0200 Subject: [PATCH 23/33] fix get assertion id contidionals (#2598) --- .../get/sender/get-assertion-id-command.js | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/src/commands/protocols/get/sender/get-assertion-id-command.js b/src/commands/protocols/get/sender/get-assertion-id-command.js index ca7629c45..6af31ae8b 100644 --- a/src/commands/protocols/get/sender/get-assertion-id-command.js +++ b/src/commands/protocols/get/sender/get-assertion-id-command.js @@ -27,14 +27,13 @@ class GetAssertionIdCommand extends Command { if ( (pendingState !== ZERO_BYTES32 && pendingState !== state) || - (pendingState === ZERO_BYTES32 && - !( - await this.blockchainModuleManager.getAssertionIds( - blockchain, - contract, - tokenId, - ) - ).includes(state)) + !( + await this.blockchainModuleManager.getAssertionIds( + blockchain, + contract, + tokenId, + ) + ).includes(state) ) { await this.handleError( operationId, @@ -56,7 +55,7 @@ class GetAssertionIdCommand extends Command { tokenId, ); } - if (assertionId == null || parseInt(assertionId, 16) === 0) { + if (assertionId == null || assertionId === ZERO_BYTES32) { assertionId = await this.blockchainModuleManager.getLatestAssertionId( blockchain, contract, From b4e7ba545e9473420a8591261d822201c4054b3c Mon Sep 17 00:00:00 2001 From: zeroxbt <89495162+zeroxbt@users.noreply.github.com> Date: Tue, 20 Jun 2023 17:09:11 +0200 Subject: [PATCH 24/33] bump version (#2599) --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 7ad232345..6592eda43 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "origintrail_node", - "version": "6.0.11", + "version": "6.0.11+hotfix.2", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "origintrail_node", - "version": "6.0.11", + "version": "6.0.11+hotfix.2", "license": "ISC", "dependencies": { "@comunica/query-sparql": "^2.4.3", diff --git a/package.json b/package.json index aaeca8c03..9baf947c6 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "origintrail_node", - "version": "6.0.11", + "version": "6.0.11+hotfix.2", "description": "OTNode V6", "main": "index.js", "type": "module", From 2860a6cb2f05070c48694b376d002a7fbf71ea0f Mon Sep 17 00:00:00 2001 From: Uladzislau Hubar Date: Wed, 21 Jun 2023 08:19:21 +0100 Subject: [PATCH 25/33] Fixed state hash validation in GetAssertionId command --- src/commands/protocols/get/sender/get-assertion-id-command.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/commands/protocols/get/sender/get-assertion-id-command.js b/src/commands/protocols/get/sender/get-assertion-id-command.js index 6af31ae8b..620a23f9d 100644 --- a/src/commands/protocols/get/sender/get-assertion-id-command.js +++ b/src/commands/protocols/get/sender/get-assertion-id-command.js @@ -26,7 +26,8 @@ class GetAssertionIdCommand extends Command { ); if ( - (pendingState !== ZERO_BYTES32 && pendingState !== state) || + state !== ZERO_BYTES32 && + state !== pendingState && !( await this.blockchainModuleManager.getAssertionIds( blockchain, From 88a5328eeab9f1e8888dff57acebc9356b05eb91 Mon Sep 17 00:00:00 2001 From: Uladzislau Hubar Date: Wed, 21 Jun 2023 08:34:46 +0100 Subject: [PATCH 26/33] Bumped the version --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 6592eda43..59757a8d5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "origintrail_node", - "version": "6.0.11+hotfix.2", + "version": "6.0.11+hotfix.3", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "origintrail_node", - "version": "6.0.11+hotfix.2", + "version": "6.0.11+hotfix.3", "license": "ISC", "dependencies": { "@comunica/query-sparql": "^2.4.3", diff --git a/package.json b/package.json index 9baf947c6..c8e8c8b86 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "origintrail_node", - "version": "6.0.11+hotfix.2", + "version": "6.0.11+hotfix.3", "description": "OTNode V6", "main": "index.js", "type": "module", From fcfcf0597011f60ed8abb49068fa2361d69e7de6 Mon Sep 17 00:00:00 2001 From: Uladzislau Hubar Date: Wed, 21 Jun 2023 09:30:06 +0100 Subject: [PATCH 27/33] Improvement for state hash validation in GetAssertionIdCommand --- .../get/sender/get-assertion-id-command.js | 28 +++++++++++-------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/src/commands/protocols/get/sender/get-assertion-id-command.js b/src/commands/protocols/get/sender/get-assertion-id-command.js index 620a23f9d..19b743d8a 100644 --- a/src/commands/protocols/get/sender/get-assertion-id-command.js +++ b/src/commands/protocols/get/sender/get-assertion-id-command.js @@ -20,22 +20,27 @@ class GetAssertionIdCommand extends Command { let assertionId; if (!Object.values(GET_STATES).includes(state)) { + if (state === ZERO_BYTES32) { + await this.handleError( + operationId, + `Given state: ${state}. State cannot be 0x0.`, + this.errorType, + ); + + return Command.empty(); + } + const pendingState = await this.blockchainModuleManager.getUnfinalizedAssertionId( blockchain, tokenId, ); + const assetStates = await this.blockchainModuleManager.getAssertionIds( + blockchain, + contract, + tokenId, + ); - if ( - state !== ZERO_BYTES32 && - state !== pendingState && - !( - await this.blockchainModuleManager.getAssertionIds( - blockchain, - contract, - tokenId, - ) - ).includes(state) - ) { + if (state !== pendingState && !assetStates.includes(state)) { await this.handleError( operationId, `Given state: ${state} doesn't exist on ${blockchain} on contract: ${contract} for Knowledge Asset with tokenId: ${tokenId}`, @@ -44,6 +49,7 @@ class GetAssertionIdCommand extends Command { return Command.empty(); } + assertionId = state; } else { this.logger.debug( From 2239dd181ec9a07ee6c9d7950e5e2ed5c0acc235 Mon Sep 17 00:00:00 2001 From: Uladzislau Hubar Date: Wed, 21 Jun 2023 09:34:10 +0100 Subject: [PATCH 28/33] Updated error messages for GetAssertionIdError --- src/commands/protocols/get/sender/get-assertion-id-command.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/commands/protocols/get/sender/get-assertion-id-command.js b/src/commands/protocols/get/sender/get-assertion-id-command.js index 19b743d8a..c2e2adc6b 100644 --- a/src/commands/protocols/get/sender/get-assertion-id-command.js +++ b/src/commands/protocols/get/sender/get-assertion-id-command.js @@ -23,7 +23,7 @@ class GetAssertionIdCommand extends Command { if (state === ZERO_BYTES32) { await this.handleError( operationId, - `Given state: ${state}. State cannot be 0x0.`, + `The provided state: ${state}. State hash cannot be 0x0.`, this.errorType, ); @@ -43,7 +43,7 @@ class GetAssertionIdCommand extends Command { if (state !== pendingState && !assetStates.includes(state)) { await this.handleError( operationId, - `Given state: ${state} doesn't exist on ${blockchain} on contract: ${contract} for Knowledge Asset with tokenId: ${tokenId}`, + `The provided state: ${state} does not exist on the ${blockchain} blockchain, ``within contract: ${contract}, for the Knowledge Asset with tokenId: ${tokenId}.`, this.errorType, ); From 0f19f745d4414ebcce69db014671cd417b220be2 Mon Sep 17 00:00:00 2001 From: Uladzislau Hubar Date: Wed, 21 Jun 2023 09:59:10 +0100 Subject: [PATCH 29/33] Changed version to 6.0.11 --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 59757a8d5..7ad232345 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "origintrail_node", - "version": "6.0.11+hotfix.3", + "version": "6.0.11", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "origintrail_node", - "version": "6.0.11+hotfix.3", + "version": "6.0.11", "license": "ISC", "dependencies": { "@comunica/query-sparql": "^2.4.3", diff --git a/package.json b/package.json index c8e8c8b86..aaeca8c03 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "origintrail_node", - "version": "6.0.11+hotfix.3", + "version": "6.0.11", "description": "OTNode V6", "main": "index.js", "type": "module", From 161f6868e274b14cf28943fe432feb2fa7162723 Mon Sep 17 00:00:00 2001 From: Uladzislau Hubar Date: Wed, 21 Jun 2023 10:33:27 +0100 Subject: [PATCH 30/33] Moved getting assertionIds array to the condition to avoid unnecessary blockchain calls --- .../get/sender/get-assertion-id-command.js | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/src/commands/protocols/get/sender/get-assertion-id-command.js b/src/commands/protocols/get/sender/get-assertion-id-command.js index c2e2adc6b..b4af9d93d 100644 --- a/src/commands/protocols/get/sender/get-assertion-id-command.js +++ b/src/commands/protocols/get/sender/get-assertion-id-command.js @@ -34,13 +34,17 @@ class GetAssertionIdCommand extends Command { blockchain, tokenId, ); - const assetStates = await this.blockchainModuleManager.getAssertionIds( - blockchain, - contract, - tokenId, - ); - if (state !== pendingState && !assetStates.includes(state)) { + if ( + state !== pendingState && + !( + await this.blockchainModuleManager.getAssertionIds( + blockchain, + contract, + tokenId, + ) + ).includes(state) + ) { await this.handleError( operationId, `The provided state: ${state} does not exist on the ${blockchain} blockchain, ``within contract: ${contract}, for the Knowledge Asset with tokenId: ${tokenId}.`, From 080d920408118720e0b94f0c6659ef089d1279f6 Mon Sep 17 00:00:00 2001 From: zeroxbt <89495162+zeroxbt@users.noreply.github.com> Date: Thu, 22 Jun 2023 12:10:36 +0200 Subject: [PATCH 31/33] fix assetHasPendingState (#2612) --- .../get/receiver/v1.0.0/v1-0-0-handle-get-init-command.js | 1 - 1 file changed, 1 deletion(-) diff --git a/src/commands/protocols/get/receiver/v1.0.0/v1-0-0-handle-get-init-command.js b/src/commands/protocols/get/receiver/v1.0.0/v1-0-0-handle-get-init-command.js index eb8edd19d..e8bcd7aee 100644 --- a/src/commands/protocols/get/receiver/v1.0.0/v1-0-0-handle-get-init-command.js +++ b/src/commands/protocols/get/receiver/v1.0.0/v1-0-0-handle-get-init-command.js @@ -42,7 +42,6 @@ class HandleGetInitCommand extends HandleProtocolMessageCommand { blockchain, contract, tokenId, - operationId, ); } From 2498202f2f03c267a922988594237e2baf060814 Mon Sep 17 00:00:00 2001 From: zeroxbt <89495162+zeroxbt@users.noreply.github.com> Date: Thu, 22 Jun 2023 12:12:58 +0200 Subject: [PATCH 32/33] bump version (#2613) --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 7ad232345..59757a8d5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "origintrail_node", - "version": "6.0.11", + "version": "6.0.11+hotfix.3", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "origintrail_node", - "version": "6.0.11", + "version": "6.0.11+hotfix.3", "license": "ISC", "dependencies": { "@comunica/query-sparql": "^2.4.3", diff --git a/package.json b/package.json index aaeca8c03..c8e8c8b86 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "origintrail_node", - "version": "6.0.11", + "version": "6.0.11+hotfix.3", "description": "OTNode V6", "main": "index.js", "type": "module", From ac717538c873af9ed047af060b224f652d695046 Mon Sep 17 00:00:00 2001 From: zeroxbt <89495162+zeroxbt@users.noreply.github.com> Date: Thu, 22 Jun 2023 13:12:15 +0200 Subject: [PATCH 33/33] bump version to 6.0.11 (#2618) --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 59757a8d5..7ad232345 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "origintrail_node", - "version": "6.0.11+hotfix.3", + "version": "6.0.11", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "origintrail_node", - "version": "6.0.11+hotfix.3", + "version": "6.0.11", "license": "ISC", "dependencies": { "@comunica/query-sparql": "^2.4.3", diff --git a/package.json b/package.json index c8e8c8b86..aaeca8c03 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "origintrail_node", - "version": "6.0.11+hotfix.3", + "version": "6.0.11", "description": "OTNode V6", "main": "index.js", "type": "module",