From 8995896a9c1ee052ead930212707d3ce7efcb7b8 Mon Sep 17 00:00:00 2001 From: zeroxbt <89495162+zeroxbt@users.noreply.github.com> Date: Sun, 14 May 2023 22:17:21 +0200 Subject: [PATCH 01/16] refactor repository module (#2514) * refactor repository module * remove duplicate code * fix table names --- .../blockchain-event-repository.js | 60 ++ .../repositories/blockchain-repository.js | 33 ++ .../repositories/command-repository.js | 54 ++ .../repositories/event-repository.js | 79 +++ .../repositories/operation-id-repository.js | 39 ++ .../repositories/operation-repository.js | 35 ++ .../repositories/operation-response.js | 30 + .../service-agreement-repository.js | 71 +++ .../repositories/shard-repository.js | 147 +++++ .../repositories/token-repository.js | 39 ++ .../sequelize/repositories/user-repository.js | 16 + .../sequelize/sequelize-repository.js | 529 +----------------- .../repository/repository-module-manager.js | 379 +++++-------- 13 files changed, 768 insertions(+), 743 deletions(-) create mode 100644 src/modules/repository/implementation/sequelize/repositories/blockchain-event-repository.js create mode 100644 src/modules/repository/implementation/sequelize/repositories/blockchain-repository.js create mode 100644 src/modules/repository/implementation/sequelize/repositories/command-repository.js create mode 100644 src/modules/repository/implementation/sequelize/repositories/event-repository.js create mode 100644 src/modules/repository/implementation/sequelize/repositories/operation-id-repository.js create mode 100644 src/modules/repository/implementation/sequelize/repositories/operation-repository.js create mode 100644 src/modules/repository/implementation/sequelize/repositories/operation-response.js create mode 100644 src/modules/repository/implementation/sequelize/repositories/service-agreement-repository.js create mode 100644 src/modules/repository/implementation/sequelize/repositories/shard-repository.js create mode 100644 src/modules/repository/implementation/sequelize/repositories/token-repository.js create mode 100644 src/modules/repository/implementation/sequelize/repositories/user-repository.js diff --git a/src/modules/repository/implementation/sequelize/repositories/blockchain-event-repository.js b/src/modules/repository/implementation/sequelize/repositories/blockchain-event-repository.js new file mode 100644 index 000000000..4e9b42c8b --- /dev/null +++ b/src/modules/repository/implementation/sequelize/repositories/blockchain-event-repository.js @@ -0,0 +1,60 @@ +import Sequelize from 'sequelize'; + +class BlockchainEventRepository { + constructor(models) { + this.sequelize = models.sequelize; + this.model = models.blockchain_event; + } + + async insertBlockchainEvents(events) { + const inserted = await this.model.bulkCreate( + events.map((event) => ({ + contract: event.contract, + event: event.event, + data: event.data, + block: event.block, + blockchain_id: event.blockchainId, + processed: false, + })), + { + ignoreDuplicates: true, + }, + ); + return inserted.map((event) => event.dataValues); + } + + async getAllUnprocessedBlockchainEvents(eventNames) { + return this.model.findAll({ + where: { + processed: false, + event: { [Sequelize.Op.in]: eventNames }, + }, + order: [['block', 'asc']], + }); + } + + async blockchainEventExists(contract, event, data, block, blockchainId) { + const dbEvent = await this.model.findOne({ + where: { + contract, + event, + data, + block, + blockchain_id: blockchainId, + }, + }); + return !!dbEvent; + } + + async markBlockchainEventsAsProcessed(events) { + const idsForUpdate = events.map((event) => event.id); + return this.model.update( + { processed: true }, + { + where: { id: { [Sequelize.Op.in]: idsForUpdate } }, + }, + ); + } +} + +export default BlockchainEventRepository; diff --git a/src/modules/repository/implementation/sequelize/repositories/blockchain-repository.js b/src/modules/repository/implementation/sequelize/repositories/blockchain-repository.js new file mode 100644 index 000000000..a07e463fb --- /dev/null +++ b/src/modules/repository/implementation/sequelize/repositories/blockchain-repository.js @@ -0,0 +1,33 @@ +class BlockchainRepository { + constructor(models) { + this.sequelize = models.sequelize; + this.model = models.blockchain; + } + + async getLastCheckedBlock(blockchainId, contract) { + return this.model.findOne({ + attributes: ['last_checked_block', 'last_checked_timestamp'], + where: { blockchain_id: blockchainId, contract }, + raw: true, + }); + } + + async removeLastCheckedBlockForContract(contract) { + return this.model.destroy({ + where: { + contract, + }, + }); + } + + async updateLastCheckedBlock(blockchainId, currentBlock, timestamp, contract) { + return this.model.upsert({ + blockchain_id: blockchainId, + contract, + last_checked_block: currentBlock, + last_checked_timestamp: timestamp, + }); + } +} + +export default BlockchainRepository; diff --git a/src/modules/repository/implementation/sequelize/repositories/command-repository.js b/src/modules/repository/implementation/sequelize/repositories/command-repository.js new file mode 100644 index 000000000..5ad957f25 --- /dev/null +++ b/src/modules/repository/implementation/sequelize/repositories/command-repository.js @@ -0,0 +1,54 @@ +import Sequelize from 'sequelize'; + +class CommandRepository { + constructor(models) { + this.sequelize = models.sequelize; + this.model = models.commands; + } + + async updateCommand(update, opts) { + await this.model.update(update, opts); + } + + async destroyCommand(name) { + await this.model.destroy({ + where: { + name: { [Sequelize.Op.eq]: name }, + }, + }); + } + + async createCommand(command, opts) { + return this.model.create(command, opts); + } + + async getCommandsWithStatus(statusArray, excludeNameArray) { + return this.model.findAll({ + where: { + status: { + [Sequelize.Op.in]: statusArray, + }, + name: { [Sequelize.Op.notIn]: excludeNameArray }, + }, + }); + } + + async getCommandWithId(id) { + return this.model.findOne({ + where: { + id, + }, + }); + } + + async removeFinalizedCommands(finalizedStatuses) { + await this.model.destroy({ + where: { + status: { [Sequelize.Op.in]: finalizedStatuses }, + started_at: { [Sequelize.Op.lte]: Date.now() }, + }, + }); + } +} + +export default CommandRepository; diff --git a/src/modules/repository/implementation/sequelize/repositories/event-repository.js b/src/modules/repository/implementation/sequelize/repositories/event-repository.js new file mode 100644 index 000000000..2e342953f --- /dev/null +++ b/src/modules/repository/implementation/sequelize/repositories/event-repository.js @@ -0,0 +1,79 @@ +import Sequelize from 'sequelize'; +import { + OPERATION_ID_STATUS, + HIGH_TRAFFIC_OPERATIONS_NUMBER_PER_HOUR, + SEND_TELEMETRY_COMMAND_FREQUENCY_MINUTES, +} from '../../../../../constants/constants.js'; + +class EventRepository { + constructor(models) { + this.sequelize = models.sequelize; + this.model = models.event; + } + + async createEventRecord(operationId, name, timestamp, value1, value2, value3) { + return this.model.create({ + operation_id: operationId, + name, + timestamp, + value1, + value2, + value3, + }); + } + + async getUnpublishedEvents() { + // events without COMPLETE/FAILED status which are older than 30min + // are also considered finished + const minutes = 5; + + let operationIds = await this.model.findAll({ + raw: true, + attributes: [ + Sequelize.fn('DISTINCT', Sequelize.col('operation_id')), + Sequelize.col('timestamp'), + ], + where: { + [Sequelize.Op.or]: { + name: { + [Sequelize.Op.in]: [ + OPERATION_ID_STATUS.COMPLETED, + OPERATION_ID_STATUS.FAILED, + ], + }, + timestamp: { + [Sequelize.Op.lt]: Sequelize.literal( + `(UNIX_TIMESTAMP()*1000 - 1000*60*${minutes})`, + ), + }, + }, + }, + order: [['timestamp', 'asc']], + limit: + Math.floor(HIGH_TRAFFIC_OPERATIONS_NUMBER_PER_HOUR / 60) * + SEND_TELEMETRY_COMMAND_FREQUENCY_MINUTES, + }); + + operationIds = operationIds.map((e) => e.operation_id); + + return this.model.findAll({ + where: { + operation_id: { + [Sequelize.Op.in]: operationIds, + }, + }, + }); + } + + async destroyEvents(ids) { + await this.model.destroy({ + where: { + id: { + [Sequelize.Op.in]: ids, + }, + }, + }); + } +} + +export default EventRepository; diff --git a/src/modules/repository/implementation/sequelize/repositories/operation-id-repository.js b/src/modules/repository/implementation/sequelize/repositories/operation-id-repository.js new file mode 100644 index 000000000..cb1664fa2 --- /dev/null +++ b/src/modules/repository/implementation/sequelize/repositories/operation-id-repository.js @@ -0,0 +1,39 @@ +import Sequelize from 'sequelize'; + +class OperationIdRepository { + constructor(models) { + this.sequelize = models.sequelize; + this.model = models.operation_ids; + } + + async createOperationIdRecord(handlerData) { + return this.model.create(handlerData); + } + + async getOperationIdRecord(operationId) { + return this.model.findOne({ + where: { + operation_id: operationId, + }, + }); + } + + async updateOperationIdRecord(data, operationId) { + await this.model.update(data, { + where: { + operation_id: operationId, + }, + }); + } + + async removeOperationIdRecord(timeToBeDeleted, statuses) { + await this.model.destroy({ + where: { + timestamp: { [Sequelize.Op.lt]: timeToBeDeleted }, + status: { [Sequelize.Op.in]: statuses }, + }, + }); + } +} + +export default OperationIdRepository; diff --git a/src/modules/repository/implementation/sequelize/repositories/operation-repository.js b/src/modules/repository/implementation/sequelize/repositories/operation-repository.js new file mode 100644 index 000000000..6d5d2a42e --- /dev/null +++ b/src/modules/repository/implementation/sequelize/repositories/operation-repository.js @@ -0,0 +1,35 @@ +class OperationRepository { + constructor(models) { + this.sequelize = models.sequelize; + this.models = { get: models.get, publish: models.publish, update: models.update }; + } + + async createOperationRecord(operation, operationId, status) { + return this.models[operation].create({ + operation_id: operationId, + status, + }); + } + + async getOperationStatus(operation, operationId) { + return this.models[operation].findOne({ + attributes: ['status'], + where: { + operation_id: operationId, + }, + }); + } + + async updateOperationStatus(operation, operationId, status) { + await this.models[operation].update( + { status }, + { + where: { + operation_id: operationId, + }, + }, + ); + } +} + +export default OperationRepository; diff --git a/src/modules/repository/implementation/sequelize/repositories/operation-response.js b/src/modules/repository/implementation/sequelize/repositories/operation-response.js new file mode 100644 index 000000000..b72d9c880 --- /dev/null +++ b/src/modules/repository/implementation/sequelize/repositories/operation-response.js @@ -0,0 +1,30 @@ +class OperationResponseRepository { + constructor(models) { + this.sequelize = models.sequelize; + this.models = { + get_response: models.get_response, + publish_response: models.publish_response, + update_response: models.update_response, + }; + } + + async createOperationResponseRecord(status, operation, operationId, keyword, message) { + await this.models[`${operation}_response`].create({ + status, + message, + operation_id: operationId, + keyword, + }); + } + + async getOperationResponsesStatuses(operation, operationId) { + return this.models[`${operation}_response`].findAll({ + attributes: ['status', 'keyword'], + where: { + operation_id: operationId, + }, + }); + } +} + +export default OperationResponseRepository; diff --git a/src/modules/repository/implementation/sequelize/repositories/service-agreement-repository.js b/src/modules/repository/implementation/sequelize/repositories/service-agreement-repository.js new file mode 100644 index 000000000..8a5fde943 --- /dev/null +++ b/src/modules/repository/implementation/sequelize/repositories/service-agreement-repository.js @@ -0,0 +1,71 @@ +import Sequelize from 'sequelize'; + +class ServiceAgreementRepository { + constructor(models) { + this.sequelize = models.sequelize; + this.model = models.service_agreement; + } + + async updateServiceAgreementRecord( + blockchainId, + contract, + tokenId, + agreementId, + startTime, + epochsNumber, + epochLength, + scoreFunctionId, + proofWindowOffsetPerc, + hashFunctionId, + keyword, + assertionId, + stateIndex, + lastCommitEpoch, + lastProofEpoch, + ) { + return this.model.upsert({ + blockchain_id: blockchainId, + asset_storage_contract_address: contract, + token_id: tokenId, + agreement_id: agreementId, + start_time: startTime, + epochs_number: epochsNumber, + epoch_length: epochLength, + score_function_id: scoreFunctionId, + proof_window_offset_perc: proofWindowOffsetPerc, + last_commit_epoch: lastCommitEpoch, + last_proof_epoch: lastProofEpoch, + hash_function_id: hashFunctionId, + keyword, + assertion_id: assertionId, + state_index: stateIndex, + }); + } + + async removeServiceAgreementRecord(blockchainId, contract, tokenId) { + await this.model.destroy({ + where: { + blockchain_id: blockchainId, + asset_storage_contract_address: contract, + token_id: tokenId, + }, + }); + } + + async updateServiceAgreementEpochsNumber(agreementId, epochsNumber) { + return this.model.update( + { epochs_number: epochsNumber }, + { + where: { agreement_id: agreementId }, + }, + ); + } + + async removeServiceAgreements(agreementIds) { + return this.model.destroy({ + where: { agreement_id: { [Sequelize.Op.in]: agreementIds } }, + }); + } +} + +export default ServiceAgreementRepository; diff --git a/src/modules/repository/implementation/sequelize/repositories/shard-repository.js b/src/modules/repository/implementation/sequelize/repositories/shard-repository.js new file mode 100644 index 000000000..6c7d9af8e --- /dev/null +++ b/src/modules/repository/implementation/sequelize/repositories/shard-repository.js @@ -0,0 +1,147 @@ +import Sequelize from 'sequelize'; + +class ShardRepository { + constructor(models) { + this.sequelize = models.sequelize; + this.model = models.shard; + } + + async createManyPeerRecords(peers) { + return this._bulkUpdatePeerRecords(peers, ['ask', 'stake', 'sha256']); + } + + async _bulkUpdatePeerRecords(peerRecords, updateColumns) { + return this.model.bulkCreate( + peerRecords.map((peerRecord) => ({ + ask: 0, + stake: 0, + sha256: '', + ...peerRecord, + })), + { + validate: true, + updateOnDuplicate: updateColumns, + }, + ); + } + + async removeShardingTablePeerRecords(blockchain) { + return this.model.destroy({ + where: { blockchain_id: blockchain }, + }); + } + + async createPeerRecord(peerId, blockchain, ask, stake, lastSeen, sha256) { + return this.model.create( + { + peer_id: peerId, + blockchain_id: blockchain, + ask, + stake, + last_seen: lastSeen, + sha256, + }, + { + ignoreDuplicates: true, + }, + ); + } + + async getAllPeerRecords(blockchain, filterLastSeen) { + const query = { + where: { + blockchain_id: { + [Sequelize.Op.eq]: blockchain, + }, + }, + raw: true, + }; + + if (filterLastSeen) { + query.where.last_seen = { + [Sequelize.Op.gte]: Sequelize.col('last_dialed'), + }; + } + + return this.model.findAll(query); + } + + async getPeerRecord(peerId, blockchain) { + return this.model.findOne({ + where: { + blockchain_id: { + [Sequelize.Op.eq]: blockchain, + }, + peer_id: { + [Sequelize.Op.eq]: peerId, + }, + }, + raw: true, + }); + } + + async getPeersCount(blockchain) { + return this.model.count({ + where: { + blockchain_id: blockchain, + }, + }); + } + + async getPeersToDial(limit, dialFrequencyMillis) { + return this.model.findAll({ + attributes: ['peer_id'], + where: { + last_dialed: { + [Sequelize.Op.lt]: new Date(Date.now() - dialFrequencyMillis), + }, + }, + order: [['last_dialed', 'asc']], + limit, + raw: true, + }); + } + + async updatePeersAsk(peerRecords) { + return this._bulkUpdatePeerRecords(peerRecords, ['ask']); + } + + async updatePeersStake(peerRecords) { + return this._bulkUpdatePeerRecords(peerRecords, ['stake']); + } + + async updatePeerRecordLastDialed(peerId, timestamp) { + await this.model.update( + { + last_dialed: timestamp, + }, + { + where: { peer_id: peerId }, + }, + ); + } + + async updatePeerRecordLastSeenAndLastDialed(peerId, timestamp) { + await this.model.update( + { + last_dialed: timestamp, + last_seen: timestamp, + }, + { + where: { peer_id: peerId }, + }, + ); + } + + async removePeerRecords(peerRecords) { + await this.model.bulkDestroy(peerRecords); + } + + async cleanShardingTable(blockchainId) { + await this.model.destroy({ + where: blockchainId ? { blockchain_id: blockchainId } : {}, + }); + } +} + +export default ShardRepository; diff --git a/src/modules/repository/implementation/sequelize/repositories/token-repository.js b/src/modules/repository/implementation/sequelize/repositories/token-repository.js new file mode 100644 index 000000000..dc0015f20 --- /dev/null +++ b/src/modules/repository/implementation/sequelize/repositories/token-repository.js @@ -0,0 +1,39 @@ +import Sequelize from 'sequelize'; + +class TokenRepository { + constructor(models) { + this.sequelize = models.sequelize; + this.model = models.token; + } + + async saveToken(tokenId, userId, tokenName, expiresAt) { + return this.model.create({ + id: tokenId, + user_id: userId, + expires_at: expiresAt, + name: tokenName, + }); + } + + async isTokenRevoked(tokenId) { + const token = await this.model.findByPk(tokenId); + + return token && token.revoked; + } + + async getTokenAbilities(tokenId) { + const abilities = await this.sequelize.query( + `SELECT a.name FROM token t + INNER JOIN user u ON t.user_id = u.id + INNER JOIN role r ON u.role_id = u.id + INNER JOIN role_ability ra on r.id = ra.role_id + INNER JOIN ability a on ra.ability_id = a.id + WHERE t.id=$tokenId;`, + { bind: { tokenId }, type: Sequelize.QueryTypes.SELECT }, + ); + + return abilities.map((e) => e.name); + } +} + +export default TokenRepository; diff --git a/src/modules/repository/implementation/sequelize/repositories/user-repository.js b/src/modules/repository/implementation/sequelize/repositories/user-repository.js new file mode 100644 index 000000000..c54a84a15 --- /dev/null +++ b/src/modules/repository/implementation/sequelize/repositories/user-repository.js @@ -0,0 +1,16 @@ +class UserRepository { + constructor(models) { + this.sequelize = models.sequelize; + this.model = models.user; + } + + async getUser(username) { + return this.model.findOne({ + where: { + name: username, + }, + }); + } +} + +export default UserRepository; diff --git a/src/modules/repository/implementation/sequelize/sequelize-repository.js b/src/modules/repository/implementation/sequelize/sequelize-repository.js index 04a2f800b..2953c6770 100644 --- a/src/modules/repository/implementation/sequelize/sequelize-repository.js +++ b/src/modules/repository/implementation/sequelize/sequelize-repository.js @@ -3,12 +3,18 @@ import path from 'path'; import fs from 'fs'; import Sequelize from 'sequelize'; import { fileURLToPath } from 'url'; -import { - OPERATION_ID_STATUS, - HIGH_TRAFFIC_OPERATIONS_NUMBER_PER_HOUR, - SEND_TELEMETRY_COMMAND_FREQUENCY_MINUTES, -} from '../../../../constants/constants.js'; import createMigrator from './sequelize-migrator.js'; +import BlockchainEventRepository from './repositories/blockchain-event-repository.js'; +import BlockchainRepository from './repositories/blockchain-repository.js'; +import CommandRepository from './repositories/command-repository.js'; +import EventRepository from './repositories/event-repository.js'; +import OperationIdRepository from './repositories/operation-id-repository.js'; +import OperationRepository from './repositories/operation-repository.js'; +import OperationResponseRepository from './repositories/operation-response.js'; +import ServiceAgreementRepository from './repositories/service-agreement-repository.js'; +import ShardRepository from './repositories/shard-repository.js'; +import TokenRepository from './repositories/token-repository.js'; +import UserRepository from './repositories/user-repository.js'; const __dirname = fileURLToPath(new URL('.', import.meta.url)); @@ -22,6 +28,20 @@ class SequelizeRepository { this.initializeSequelize(); await this.runMigrations(); await this.loadModels(); + + this.repositories = { + blockchain_event: new BlockchainEventRepository(this.models), + blockchain: new BlockchainRepository(this.models), + command: new CommandRepository(this.models), + event: new EventRepository(this.models), + operation_id: new OperationIdRepository(this.models), + operation: new OperationRepository(this.models), + operation_response: new OperationResponseRepository(this.models), + service_agreement: new ServiceAgreementRepository(this.models), + shard: new ShardRepository(this.models), + token: new TokenRepository(this.models), + user: new UserRepository(this.models), + }; } initializeSequelize() { @@ -101,508 +121,13 @@ class SequelizeRepository { return this.models.sequelize.transaction(async (t) => execFn(t)); } - async updateServiceAgreementRecord( - blockchainId, - contract, - tokenId, - agreementId, - startTime, - epochsNumber, - epochLength, - scoreFunctionId, - proofWindowOffsetPerc, - hashFunctionId, - keyword, - assertionId, - stateIndex, - lastCommitEpoch, - lastProofEpoch, - ) { - return this.models.service_agreement.upsert({ - blockchain_id: blockchainId, - asset_storage_contract_address: contract, - token_id: tokenId, - agreement_id: agreementId, - start_time: startTime, - epochs_number: epochsNumber, - epoch_length: epochLength, - score_function_id: scoreFunctionId, - proof_window_offset_perc: proofWindowOffsetPerc, - last_commit_epoch: lastCommitEpoch, - last_proof_epoch: lastProofEpoch, - hash_function_id: hashFunctionId, - keyword, - assertion_id: assertionId, - state_index: stateIndex, - }); - } - - async removeServiceAgreementRecord(blockchainId, contract, tokenId) { - await this.models.service_agreement.destroy({ - where: { - blockchain_id: blockchainId, - asset_storage_contract_address: contract, - token_id: tokenId, - }, - }); - } - - // COMMAND - async updateCommand(update, opts) { - await this.models.commands.update(update, opts); - } - - async destroyCommand(name) { - await this.models.commands.destroy({ - where: { - name: { [Sequelize.Op.eq]: name }, - }, - }); - } - - async createCommand(command, opts) { - return this.models.commands.create(command, opts); - } - - async getCommandsWithStatus(statusArray, excludeNameArray) { - return this.models.commands.findAll({ - where: { - status: { - [Sequelize.Op.in]: statusArray, - }, - name: { [Sequelize.Op.notIn]: excludeNameArray }, - }, - }); - } - - async getCommandWithId(id) { - return this.models.commands.findOne({ - where: { - id, - }, - }); - } - - async removeFinalizedCommands(finalizedStatuses) { - await this.models.commands.destroy({ - where: { - status: { [Sequelize.Op.in]: finalizedStatuses }, - started_at: { [Sequelize.Op.lte]: Date.now() }, - }, - }); - } - - // OPERATION_ID - async createOperationIdRecord(handlerData) { - return this.models.operation_ids.create(handlerData); - } - - async getOperationIdRecord(operationId) { - return this.models.operation_ids.findOne({ - where: { - operation_id: operationId, - }, - }); - } - - async updateOperationIdRecord(data, operationId) { - await this.models.operation_ids.update(data, { - where: { - operation_id: operationId, - }, - }); - } - - async removeOperationIdRecord(timeToBeDeleted, statuses) { - await this.models.operation_ids.destroy({ - where: { - timestamp: { [Sequelize.Op.lt]: timeToBeDeleted }, - status: { [Sequelize.Op.in]: statuses }, - }, - }); - } - - async createOperationRecord(operation, operationId, status) { - return this.models[operation].create({ - operation_id: operationId, - status, - }); - } - - async getOperationStatus(operation, operationId) { - return this.models[operation].findOne({ - attributes: ['status'], - where: { - operation_id: operationId, - }, - }); - } - - async updateOperationStatus(operation, operationId, status) { - await this.models[operation].update( - { status }, - { - where: { - operation_id: operationId, - }, - }, - ); - } - - async createOperationResponseRecord(status, operation, operationId, keyword, message) { - await this.models[`${operation}_response`].create({ - status, - message, - operation_id: operationId, - keyword, - }); - } - - async getOperationResponsesStatuses(operation, operationId) { - return this.models[`${operation}_response`].findAll({ - attributes: ['status', 'keyword'], - where: { - operation_id: operationId, - }, - }); - } - - // Sharding Table - async createManyPeerRecords(peers) { - return this._bulkUpdatePeerRecords(peers, ['ask', 'stake', 'sha256']); - } - - async _bulkUpdatePeerRecords(peerRecords, updateColumns) { - return this.models.shard.bulkCreate( - peerRecords.map((peerRecord) => ({ - ask: 0, - stake: 0, - sha256: '', - ...peerRecord, - })), - { - validate: true, - updateOnDuplicate: updateColumns, - }, - ); - } - - async removeShardingTablePeerRecords(blockchain) { - return this.models.shard.destroy({ - where: { blockchain_id: blockchain }, - }); - } - - async createPeerRecord(peerId, blockchain, ask, stake, lastSeen, sha256) { - return this.models.shard.create( - { - peer_id: peerId, - blockchain_id: blockchain, - ask, - stake, - last_seen: lastSeen, - sha256, - }, - { - ignoreDuplicates: true, - }, - ); - } - - async getAllPeerRecords(blockchain, filterLastSeen) { - const query = { - where: { - blockchain_id: { - [Sequelize.Op.eq]: blockchain, - }, - }, - raw: true, - }; - - if (filterLastSeen) { - query.where.last_seen = { - [Sequelize.Op.gte]: Sequelize.col('last_dialed'), - }; - } - - return this.models.shard.findAll(query); - } - - async getPeerRecord(peerId, blockchain) { - return this.models.shard.findOne({ - where: { - blockchain_id: { - [Sequelize.Op.eq]: blockchain, - }, - peer_id: { - [Sequelize.Op.eq]: peerId, - }, - }, - raw: true, - }); - } - - async getPeersCount(blockchain) { - return this.models.shard.count({ - where: { - blockchain_id: blockchain, - }, - }); - } - - async getPeersToDial(limit, dialFrequencyMillis) { - return this.models.shard.findAll({ - attributes: ['peer_id'], - where: { - last_dialed: { - [Sequelize.Op.lt]: new Date(Date.now() - dialFrequencyMillis), - }, - }, - order: [['last_dialed', 'asc']], - limit, - raw: true, - }); - } - - async updatePeersAsk(peerRecords) { - return this._bulkUpdatePeerRecords(peerRecords, ['ask']); - } - - async updatePeersStake(peerRecords) { - return this._bulkUpdatePeerRecords(peerRecords, ['stake']); - } - - async updatePeerRecordLastDialed(peerId, timestamp) { - await this.models.shard.update( - { - last_dialed: timestamp, - }, - { - where: { peer_id: peerId }, - }, - ); - } - - async updatePeerRecordLastSeenAndLastDialed(peerId, timestamp) { - await this.models.shard.update( - { - last_dialed: timestamp, - last_seen: timestamp, - }, - { - where: { peer_id: peerId }, - }, - ); - } - - async removePeerRecords(peerRecords) { - await this.models.shard.bulkDestroy(peerRecords); - } - - async cleanShardingTable(blockchainId) { - await this.models.shard.destroy({ - where: blockchainId ? { blockchain_id: blockchainId } : {}, - }); - } - - async getLastCheckedBlock(blockchainId, contract) { - return this.models.blockchain.findOne({ - attributes: ['last_checked_block', 'last_checked_timestamp'], - where: { blockchain_id: blockchainId, contract }, - raw: true, - }); - } - - async removeLastCheckedBlockForContract(contract) { - return this.models.blockchain.destroy({ - where: { - contract, - }, - }); - } - - async updateLastCheckedBlock(blockchainId, currentBlock, timestamp, contract) { - return this.models.blockchain.upsert({ - blockchain_id: blockchainId, - contract, - last_checked_block: currentBlock, - last_checked_timestamp: timestamp, - }); - } - - // EVENT - async createEventRecord(operationId, name, timestamp, value1, value2, value3) { - return this.models.event.create({ - operation_id: operationId, - name, - timestamp, - value1, - value2, - value3, - }); - } - - async getUnpublishedEvents() { - // events without COMPLETE/FAILED status which are older than 30min - // are also considered finished - const minutes = 5; - - let operationIds = await this.models.event.findAll({ - raw: true, - attributes: [ - Sequelize.fn('DISTINCT', Sequelize.col('operation_id')), - Sequelize.col('timestamp'), - ], - where: { - [Sequelize.Op.or]: { - name: { - [Sequelize.Op.in]: [ - OPERATION_ID_STATUS.COMPLETED, - OPERATION_ID_STATUS.FAILED, - ], - }, - timestamp: { - [Sequelize.Op.lt]: Sequelize.literal( - `(UNIX_TIMESTAMP()*1000 - 1000*60*${minutes})`, - ), - }, - }, - }, - order: [['timestamp', 'asc']], - limit: - Math.floor(HIGH_TRAFFIC_OPERATIONS_NUMBER_PER_HOUR / 60) * - SEND_TELEMETRY_COMMAND_FREQUENCY_MINUTES, - }); - - operationIds = operationIds.map((e) => e.operation_id); - - return this.models.event.findAll({ - where: { - operation_id: { - [Sequelize.Op.in]: operationIds, - }, - }, - }); - } - - async destroyEvents(ids) { - await this.models.event.destroy({ - where: { - id: { - [Sequelize.Op.in]: ids, - }, - }, - }); - } - - async getUser(username) { - return this.models.user.findOne({ - where: { - name: username, - }, - }); - } - - async saveToken(tokenId, userId, tokenName, expiresAt) { - return this.models.token.create({ - id: tokenId, - user_id: userId, - expires_at: expiresAt, - name: tokenName, - }); - } - - async isTokenRevoked(tokenId) { - const token = await this.models.token.findByPk(tokenId); - - return token && token.revoked; - } - - async getTokenAbilities(tokenId) { - const abilities = await this.models.sequelize.query( - `SELECT a.name FROM token t - INNER JOIN user u ON t.user_id = u.id - INNER JOIN role r ON u.role_id = u.id - INNER JOIN role_ability ra on r.id = ra.role_id - INNER JOIN ability a on ra.ability_id = a.id - WHERE t.id=$tokenId;`, - { bind: { tokenId }, type: Sequelize.QueryTypes.SELECT }, - ); - - return abilities.map((e) => e.name); + getRepository(repositoryName) { + return this.repositories[repositoryName]; } async query(query) { return this.models.sequelize.query(query); } - - async insertBlockchainEvents(events) { - const inserted = await this.models.blockchain_event.bulkCreate( - events.map((event) => ({ - contract: event.contract, - event: event.event, - data: event.data, - block: event.block, - blockchain_id: event.blockchainId, - processed: false, - })), - { - ignoreDuplicates: true, - }, - ); - return inserted.map((event) => event.dataValues); - } - - async getAllUnprocessedBlockchainEvents(eventNames) { - return this.models.blockchain_event.findAll({ - where: { - processed: false, - event: { [Sequelize.Op.in]: eventNames }, - }, - order: [['block', 'asc']], - }); - } - - async blockchainEventExists(contract, event, data, block, blockchainId) { - const dbEvent = await this.models.blockchain_event.findOne({ - where: { - contract, - event, - data, - block, - blockchain_id: blockchainId, - }, - }); - return !!dbEvent; - } - - async markBlockchainEventsAsProcessed(events) { - const idsForUpdate = events.map((event) => event.id); - return this.models.blockchain_event.update( - { processed: true }, - { - where: { id: { [Sequelize.Op.in]: idsForUpdate } }, - }, - ); - } - - // eslint-disable-next-line no-empty-function - async getEligibleSubmitCommits() {} - - async updateServiceAgreementEpochsNumber(agreementId, epochsNumber) { - return this.models.service_agreement.update( - { epochs_number: epochsNumber }, - { - where: { agreement_id: agreementId }, - }, - ); - } - - async removeServiceAgreements(agreementIds) { - return this.models.service_agreement.destroy({ - where: { agreement_id: { [Sequelize.Op.in]: agreementIds } }, - }); - } } export default SequelizeRepository; diff --git a/src/modules/repository/repository-module-manager.js b/src/modules/repository/repository-module-manager.js index 5c332a84b..42f0a2bbb 100644 --- a/src/modules/repository/repository-module-manager.js +++ b/src/modules/repository/repository-module-manager.js @@ -5,6 +5,13 @@ class RepositoryModuleManager extends BaseModuleManager { return 'repository'; } + getRepository(repoName) { + if (!this.initialized) { + throw new Error('RepositoryModuleManager not initialized'); + } + return this.getImplementation().module.getRepository(repoName); + } + transaction(execFn) { if (this.initialized) { return this.getImplementation().module.transaction(execFn); @@ -17,272 +24,156 @@ class RepositoryModuleManager extends BaseModuleManager { } } - async updateServiceAgreementRecord( - blockchainId, - contract, - tokenId, - agreementId, - startTime, - epochsNumber, - epochLength, - scoreFunctionId, - proofWindowOffsetPerc, - hashFunctionId, - keyword, - assertionId, - stateIndex, - lastCommitEpoch, - lastProofEpoch, - ) { - if (this.initialized) { - return this.getImplementation().module.updateServiceAgreementRecord( - blockchainId, - contract, - tokenId, - agreementId, - startTime, - epochsNumber, - epochLength, - scoreFunctionId, - proofWindowOffsetPerc, - hashFunctionId, - keyword, - assertionId, - stateIndex, - lastCommitEpoch, - lastProofEpoch, - ); - } - } - - async removeServiceAgreementRecord(blockchainId, contract, tokenId) { + async query(query) { if (this.initialized) { - return this.getImplementation().module.removeServiceAgreementRecord( - blockchainId, - contract, - tokenId, - ); + return this.getImplementation().module.query(query); } } - // COMMANDS async updateCommand(update, opts) { - if (this.initialized) { - return this.getImplementation().module.updateCommand(update, opts); - } + return this.getRepository('command').updateCommand(update, opts); } async destroyCommand(name) { - if (this.initialized) { - return this.getImplementation().module.destroyCommand(name); - } + return this.getRepository('command').destroyCommand(name); } async createCommand(command, opts) { - if (this.initialized) { - return this.getImplementation().module.createCommand(command, opts); - } + return this.getRepository('command').createCommand(command, opts); } async getCommandsWithStatus(statusArray, excludeNameArray = []) { - if (this.initialized) { - return this.getImplementation().module.getCommandsWithStatus( - statusArray, - excludeNameArray, - ); - } + return this.getRepository('command').getCommandsWithStatus(statusArray, excludeNameArray); } async getCommandWithId(id) { - if (this.initialized) { - return this.getImplementation().module.getCommandWithId(id); - } + return this.getRepository('command').getCommandWithId(id); } async removeFinalizedCommands(finalizedStatuses) { - if (this.initialized) { - return this.getImplementation().module.removeFinalizedCommands(finalizedStatuses); - } + return this.getRepository('command').removeFinalizedCommands(finalizedStatuses); } - // OPERATION ID TABLE async createOperationIdRecord(handlerData) { - if (this.initialized) { - return this.getImplementation().module.createOperationIdRecord(handlerData); - } + return this.getRepository('operation_id').createOperationIdRecord(handlerData); } async updateOperationIdRecord(data, operationId) { - if (this.initialized) { - return this.getImplementation().module.updateOperationIdRecord(data, operationId); - } + return this.getRepository('operation_id').updateOperationIdRecord(data, operationId); } async getOperationIdRecord(operationId) { - if (this.initialized) { - return this.getImplementation().module.getOperationIdRecord(operationId); - } + return this.getRepository('operation_id').getOperationIdRecord(operationId); } async removeOperationIdRecord(timeToBeDeleted, statuses) { - if (this.initialized) { - return this.getImplementation().module.removeOperationIdRecord( - timeToBeDeleted, - statuses, - ); - } + return this.getRepository('operation_id').removeOperationIdRecord( + timeToBeDeleted, + statuses, + ); } - // publish table async createOperationRecord(operation, operationId, status) { - if (this.initialized) { - return this.getImplementation().module.createOperationRecord( - operation, - operationId, - status, - ); - } + return this.getRepository('operation').createOperationRecord( + operation, + operationId, + status, + ); } async getOperationStatus(operation, operationId) { - if (this.initialized) { - return this.getImplementation().module.getOperationStatus(operation, operationId); - } + return this.getRepository('operation').getOperationStatus(operation, operationId); } async updateOperationStatus(operation, operationId, status) { - if (this.initialized) { - return this.getImplementation().module.updateOperationStatus( - operation, - operationId, - status, - ); - } + return this.getRepository('operation').updateOperationStatus( + operation, + operationId, + status, + ); } async createOperationResponseRecord(status, operation, operationId, keyword, errorMessage) { - if (this.initialized) { - return this.getImplementation().module.createOperationResponseRecord( - status, - operation, - operationId, - keyword, - errorMessage, - ); - } + return this.getRepository('operation_response').createOperationResponseRecord( + status, + operation, + operationId, + keyword, + errorMessage, + ); } async getOperationResponsesStatuses(operation, operationId) { - if (this.initialized) { - return this.getImplementation().module.getOperationResponsesStatuses( - operation, - operationId, - ); - } + return this.getRepository('operation_response').getOperationResponsesStatuses( + operation, + operationId, + ); } // Sharding Table async createManyPeerRecords(peers) { - if (this.initialized) { - return this.getImplementation().module.createManyPeerRecords(peers); - } + return this.getRepository('shard').createManyPeerRecords(peers); } async removeShardingTablePeerRecords(blockchain) { - if (this.initialized) { - return this.getImplementation().module.removeShardingTablePeerRecords(blockchain); - } + return this.getRepository('shard').removeShardingTablePeerRecords(blockchain); } async createPeerRecord(peerId, blockchain, ask, stake, lastSeen, sha256) { - if (this.initialized) { - return this.getImplementation().module.createPeerRecord( - peerId, - blockchain, - ask, - stake, - lastSeen, - sha256, - ); - } + return this.getRepository('shard').createPeerRecord( + peerId, + blockchain, + ask, + stake, + lastSeen, + sha256, + ); } async getPeerRecord(peerId, blockchain) { - if (this.initialized) { - return this.getImplementation().module.getPeerRecord(peerId, blockchain); - } + return this.getRepository('shard').getPeerRecord(peerId, blockchain); } async getAllPeerRecords(blockchain, filterLastSeen) { - if (this.initialized) { - return this.getImplementation().module.getAllPeerRecords(blockchain, filterLastSeen); - } + return this.getRepository('shard').getAllPeerRecords(blockchain, filterLastSeen); } async getPeersCount(blockchain) { - if (this.initialized) { - return this.getImplementation().module.getPeersCount(blockchain); - } + return this.getRepository('shard').getPeersCount(blockchain); } async getPeersToDial(limit, dialFrequencyMillis) { - if (this.initialized) { - return this.getImplementation().module.getPeersToDial(limit, dialFrequencyMillis); - } - } - - async query(query) { - if (this.initialized) { - return this.getImplementation().module.query(query); - } + return this.getRepository('shard').getPeersToDial(limit, dialFrequencyMillis); } async removePeerRecords(peerRecords) { - if (this.initialized) { - return this.getImplementation().module.removePeerRecords(peerRecords); - } + return this.getRepository('shard').removePeerRecords(peerRecords); } async updatePeerRecordLastDialed(peerId, timestamp) { - if (this.initialized) { - return this.getImplementation().module.updatePeerRecordLastDialed(peerId, timestamp); - } + return this.getRepository('shard').updatePeerRecordLastDialed(peerId, timestamp); } async updatePeerRecordLastSeenAndLastDialed(peerId, timestamp) { - if (this.initialized) { - return this.getImplementation().module.updatePeerRecordLastSeenAndLastDialed( - peerId, - timestamp, - ); - } + return this.getRepository('shard').updatePeerRecordLastSeenAndLastDialed(peerId, timestamp); } async updatePeersAsk(peerRecords) { - if (this.initialized) { - return this.getImplementation().module.updatePeersAsk(peerRecords); - } + return this.getRepository('shard').updatePeersAsk(peerRecords); } async updatePeersStake(peerRecords) { - if (this.initialized) { - return this.getImplementation().module.updatePeersStake(peerRecords); - } + return this.getRepository('shard').updatePeersStake(peerRecords); } async getNeighbourhood(assertionId, r2) { - if (this.initialized) { - return this.getImplementation().module.getNeighbourhood(assertionId, r2); - } + return this.getRepository('shard').getNeighbourhood(assertionId, r2); } async cleanShardingTable(blockchainId) { - if (this.initialized) { - return this.getImplementation().module.cleanShardingTable(blockchainId); - } + return this.getRepository('shard').cleanShardingTable(blockchainId); } - // EVENT async createEventRecord( operationId, name, @@ -291,120 +182,126 @@ class RepositoryModuleManager extends BaseModuleManager { value2 = null, value3 = null, ) { - if (this.initialized) { - return this.getImplementation().module.createEventRecord( - operationId, - name, - timestamp, - value1, - value2, - value3, - ); - } + return this.getRepository('event').createEventRecord( + operationId, + name, + timestamp, + value1, + value2, + value3, + ); } async getUnpublishedEvents() { - if (this.initialized) { - return this.getImplementation().module.getUnpublishedEvents(); - } + return this.getRepository('event').getUnpublishedEvents(); } async destroyEvents(ids) { - if (this.initialized) { - return this.getImplementation().module.destroyEvents(ids); - } + return this.getRepository('event').destroyEvents(ids); } async getUser(username) { - if (this.initialized) { - return this.getImplementation().module.getUser(username); - } + return this.getRepository('user').getUser(username); } async saveToken(tokenId, userId, tokenName, expiresAt) { - if (this.initialized) { - return this.getImplementation().module.saveToken(tokenId, userId, tokenName, expiresAt); - } + return this.getRepository('token').saveToken(tokenId, userId, tokenName, expiresAt); } async isTokenRevoked(tokenId) { - if (this.initialized) { - return this.getImplementation().module.isTokenRevoked(tokenId); - } + return this.getRepository('token').isTokenRevoked(tokenId); } async getTokenAbilities(tokenId) { - if (this.initialized) { - return this.getImplementation().module.getTokenAbilities(tokenId); - } + return this.getRepository('token').getTokenAbilities(tokenId); } async insertBlockchainEvents(events) { - if (this.initialized) { - return this.getImplementation().module.insertBlockchainEvents(events); - } + return this.getRepository('blockchain_event').insertBlockchainEvents(events); } async getAllUnprocessedBlockchainEvents(eventNames) { - if (this.initialized) { - return this.getImplementation().module.getAllUnprocessedBlockchainEvents(eventNames); - } + return this.getRepository('blockchain_event').getAllUnprocessedBlockchainEvents(eventNames); } async markBlockchainEventsAsProcessed(events) { - if (this.initialized) { - return this.getImplementation().module.markBlockchainEventsAsProcessed(events); - } + return this.getRepository('blockchain_event').markBlockchainEventsAsProcessed(events); } async removeBlockchainEvents(contract) { - if (this.initialized) { - return this.getImplementation().module.removeBlockchainEvents(contract); - } + return this.getRepository('blockchain_event').removeBlockchainEvents(contract); } async removeLastCheckedBlockForContract(contract) { - if (this.initialized) { - return this.getImplementation().module.removeLastCheckedBlockForContract(contract); - } + return this.getRepository('blockchain').removeLastCheckedBlockForContract(contract); } async getLastCheckedBlock(blockchainId, contract) { - if (this.initialized) { - return this.getImplementation().module.getLastCheckedBlock(blockchainId, contract); - } + return this.getRepository('blockchain').getLastCheckedBlock(blockchainId, contract); } async updateLastCheckedBlock(blockchainId, currentBlock, timestamp, contract) { - if (this.initialized) { - return this.getImplementation().module.updateLastCheckedBlock( - blockchainId, - currentBlock, - timestamp, - contract, - ); - } + return this.getRepository('blockchain').updateLastCheckedBlock( + blockchainId, + currentBlock, + timestamp, + contract, + ); } - async getEligibleSubmitCommits() { - if (this.initialized) { - return this.getImplementation().module.getEligibleSubmitCommits(); - } + async updateServiceAgreementRecord( + blockchainId, + contract, + tokenId, + agreementId, + startTime, + epochsNumber, + epochLength, + scoreFunctionId, + proofWindowOffsetPerc, + hashFunctionId, + keyword, + assertionId, + stateIndex, + lastCommitEpoch, + lastProofEpoch, + ) { + return this.getRepository('service_agreement').updateServiceAgreementRecord( + blockchainId, + contract, + tokenId, + agreementId, + startTime, + epochsNumber, + epochLength, + scoreFunctionId, + proofWindowOffsetPerc, + hashFunctionId, + keyword, + assertionId, + stateIndex, + lastCommitEpoch, + lastProofEpoch, + ); + } + + async removeServiceAgreementRecord(blockchainId, contract, tokenId) { + return this.getRepository('service_agreement').removeServiceAgreementRecord( + blockchainId, + contract, + tokenId, + ); } async removeServiceAgreements(agreementIds) { - if (this.initialized) { - return this.getImplementation().module.removeServiceAgreements(agreementIds); - } + return this.getRepository('service_agreement').removeServiceAgreements(agreementIds); } async updateServiceAgreementEpochsNumber(agreementId, epochsNumber) { - if (this.initialized) { - return this.getImplementation().module.updateServiceAgreementEpochsNumber( - agreementId, - epochsNumber, - ); - } + return this.getRepository('service_agreement').updateServiceAgreementEpochsNumber( + agreementId, + epochsNumber, + ); } } From 31dde52953d5b72e24fd8eb8f65581247eaf48b8 Mon Sep 17 00:00:00 2001 From: zeroxbt <89495162+zeroxbt@users.noreply.github.com> Date: Tue, 16 May 2023 10:57:23 +0200 Subject: [PATCH 02/16] fix logic for getting past blockchain events (#2515) * fix logic for getting past blockchain events * fix block range calculation --- .../blockchain/implementation/web3-service.js | 56 ++++++++----------- 1 file changed, 23 insertions(+), 33 deletions(-) diff --git a/src/modules/blockchain/implementation/web3-service.js b/src/modules/blockchain/implementation/web3-service.js index ad0248095..5eea08430 100644 --- a/src/modules/blockchain/implementation/web3-service.js +++ b/src/modules/blockchain/implementation/web3-service.js @@ -397,46 +397,36 @@ class Web3Service { let fromBlock; if (this.isOlderThan(lastCheckedTimestamp, DEFAULT_BLOCKCHAIN_EVENT_SYNC_PERIOD_IN_MILLS)) { - fromBlock = this.startBlock - 10; + fromBlock = this.startBlock; } else { fromBlock = lastCheckedBlock + 1; } let events = []; - if (currentBlock - fromBlock > MAXIMUM_NUMBERS_OF_BLOCKS_TO_FETCH) { - let iteration = 1; - - while (fromBlock - MAXIMUM_NUMBERS_OF_BLOCKS_TO_FETCH > currentBlock) { - events.concat( - await contract.queryFilter( - '*', - fromBlock, - fromBlock + MAXIMUM_NUMBERS_OF_BLOCKS_TO_FETCH * iteration, - ), - ); - fromBlock += MAXIMUM_NUMBERS_OF_BLOCKS_TO_FETCH * iteration; - iteration += 1; - } - } else { - events = await contract.queryFilter('*', fromBlock, currentBlock); + while (fromBlock <= currentBlock) { + const toBlock = Math.min( + fromBlock + MAXIMUM_NUMBERS_OF_BLOCKS_TO_FETCH - 1, + currentBlock, + ); + const newEvents = await contract.queryFilter('*', fromBlock, toBlock); + events = events.concat(newEvents); + fromBlock = toBlock + 1; } - return events - ? events.map((event) => ({ - contract: contractName, - event: event.event, - data: JSON.stringify( - Object.fromEntries( - Object.entries(event.args).map(([k, v]) => [ - k, - ethers.BigNumber.isBigNumber(v) ? v.toString() : v, - ]), - ), - ), - block: event.blockNumber, - blockchainId, - })) - : []; + return events.map((event) => ({ + contract: contractName, + event: event.event, + data: JSON.stringify( + Object.fromEntries( + Object.entries(event.args).map(([k, v]) => [ + k, + ethers.BigNumber.isBigNumber(v) ? v.toString() : v, + ]), + ), + ), + block: event.blockNumber, + blockchainId, + })); } isOlderThan(timestamp, olderThanInMills) { From bcb7dd7107f867d000dfb8e58dbc23ef4b030dc1 Mon Sep 17 00:00:00 2001 From: zeroxbt <89495162+zeroxbt@users.noreply.github.com> Date: Tue, 16 May 2023 13:34:20 +0200 Subject: [PATCH 03/16] add new epoch check commands (#2492) * add repository migrations and models * add migration file * add first implementation of service agreement migration * add execute migration at start * fix typo * small fixes * update migration name and order of execution * update service agreements table on req and event * update commit and proof table on succesful txs * Updated naming of new tables * Added epoch check command * Updated query for fetching eligible service agreements for commit and proof * save last processed token id in file * bug fixes, better error handling and logs * update commit and proof table on failed tx * update migrations and queries * add repository tests * add tests, bug fixes * update Eligible service agreements test * add test last commit epoch * add bug fixes, more test scenarios * remove unnecessary tables * Updated epoch check command * update service agreement table * add last commit epoch and last proof epoch in migration * fix primary key too long, fix tests * add bug fixes to epoch check commands * bug fixes, update tests * remove unused epoch command * update migration, add load tests * update agreements for load tests * update epoch check and number of transactions * revert changes to default command delay * add better logs --------- Co-authored-by: Djordje Kovacevic --- scripts/set-ask.js | 7 +- scripts/set-operator-fee.js | 7 +- scripts/set-stake.js | 13 +- .../protocols/common/epoch-check-command.js | 298 ++++++++++++ .../protocols/common/epoch-command.js | 130 ----- .../protocols/common/submit-commit-command.js | 149 ++++++ .../submit-proofs-command.js | 142 +++--- .../receiver/calculate-proofs-command.js | 171 ------- .../publish/receiver/epoch-check-command.js | 143 ------ .../publish/receiver/submit-commit-command.js | 263 ---------- .../v1-0-0-handle-store-request-command.js | 48 +- .../receiver/submit-update-commit-command.js | 40 +- .../v1-0-0-handle-update-request-command.js | 14 +- src/constants/constants.js | 11 +- .../service-agreements-metadata-migration.js | 1 - .../blockchain/blockchain-module-manager.js | 8 + .../implementation/hardhat/hardhat-service.js | 5 + .../ot-parachain/ot-parachain-service.js | 5 + .../blockchain/implementation/web3-service.js | 17 +- .../service-agreement-repository.js | 150 +++++- .../sequelize/sequelize-repository.js | 4 + .../repository/repository-module-manager.js | 108 +++- test/modules/repository/config.json | 22 + test/modules/repository/repository.js | 460 ++++++++++++++++++ test/modules/triple-store/triple-store.js | 3 +- 25 files changed, 1359 insertions(+), 860 deletions(-) create mode 100644 src/commands/protocols/common/epoch-check-command.js delete mode 100644 src/commands/protocols/common/epoch-command.js create mode 100644 src/commands/protocols/common/submit-commit-command.js rename src/commands/protocols/{publish/receiver => common}/submit-proofs-command.js (57%) delete mode 100644 src/commands/protocols/publish/receiver/calculate-proofs-command.js delete mode 100644 src/commands/protocols/publish/receiver/epoch-check-command.js delete mode 100644 src/commands/protocols/publish/receiver/submit-commit-command.js create mode 100644 test/modules/repository/config.json create mode 100644 test/modules/repository/repository.js diff --git a/scripts/set-ask.js b/scripts/set-ask.js index 7b710e56c..71bba163d 100644 --- a/scripts/set-ask.js +++ b/scripts/set-ask.js @@ -4,6 +4,7 @@ import { createRequire } from 'module'; import { NODE_ENVIRONMENTS, TRANSACTION_POLLING_TIMEOUT_MILLIS, + TRANSACTION_CONFIRMATIONS, } from '../src/constants/constants.js'; import validateArguments from './utils.js'; @@ -39,7 +40,11 @@ async function setAsk(rpcEndpoint, ask, walletPrivateKey, hubContractAddress) { gasPrice: devEnvironment ? undefined : 8, gasLimit: 500_000, }); - await provider.waitForTransaction(tx.hash, null, TRANSACTION_POLLING_TIMEOUT_MILLIS); + await provider.waitForTransaction( + tx.hash, + TRANSACTION_CONFIRMATIONS, + TRANSACTION_POLLING_TIMEOUT_MILLIS, + ); } const expectedArguments = ['rpcEndpoint', 'ask', 'privateKey', 'hubContractAddress']; diff --git a/scripts/set-operator-fee.js b/scripts/set-operator-fee.js index 876c7ff33..f30e5887f 100644 --- a/scripts/set-operator-fee.js +++ b/scripts/set-operator-fee.js @@ -4,6 +4,7 @@ import { createRequire } from 'module'; import { NODE_ENVIRONMENTS, TRANSACTION_POLLING_TIMEOUT_MILLIS, + TRANSACTION_CONFIRMATIONS, } from '../src/constants/constants.js'; import validateArguments from './utils.js'; @@ -37,7 +38,11 @@ async function setOperatorFee(rpcEndpoint, operatorFee, walletPrivateKey, hubCon gasPrice: process.env.NODE_ENV === NODE_ENVIRONMENTS.DEVELOPMENT ? undefined : 8, gasLimit: 500_000, }); - await provider.waitForTransaction(tx.hash, null, TRANSACTION_POLLING_TIMEOUT_MILLIS); + await provider.waitForTransaction( + tx.hash, + TRANSACTION_CONFIRMATIONS, + TRANSACTION_POLLING_TIMEOUT_MILLIS, + ); } const expectedArguments = ['rpcEndpoint', 'operatorFee', 'privateKey', 'hubContractAddress']; diff --git a/scripts/set-stake.js b/scripts/set-stake.js index 5b6ef019b..953b6f1e6 100644 --- a/scripts/set-stake.js +++ b/scripts/set-stake.js @@ -4,6 +4,7 @@ import { createRequire } from 'module'; import { NODE_ENVIRONMENTS, TRANSACTION_POLLING_TIMEOUT_MILLIS, + TRANSACTION_CONFIRMATIONS, } from '../src/constants/constants.js'; import validateArguments from './utils.js'; @@ -55,13 +56,21 @@ async function setStake( gasPrice: devEnvironment ? undefined : 8, gasLimit: 500_000, }); - await provider.waitForTransaction(tx.hash, null, TRANSACTION_POLLING_TIMEOUT_MILLIS); + await provider.waitForTransaction( + tx.hash, + TRANSACTION_CONFIRMATIONS, + TRANSACTION_POLLING_TIMEOUT_MILLIS, + ); // TODO: Add ABI instead of hard-coded function definition tx = await stakingContract['addStake(uint72,uint96)'](identityId, stakeWei, { gasPrice: devEnvironment ? undefined : 1_000, gasLimit: 500_000, }); - await provider.waitForTransaction(tx.hash, null, TRANSACTION_POLLING_TIMEOUT_MILLIS); + await provider.waitForTransaction( + tx.hash, + TRANSACTION_CONFIRMATIONS, + TRANSACTION_POLLING_TIMEOUT_MILLIS, + ); } const expectedArguments = [ diff --git a/src/commands/protocols/common/epoch-check-command.js b/src/commands/protocols/common/epoch-check-command.js new file mode 100644 index 000000000..214c0cdce --- /dev/null +++ b/src/commands/protocols/common/epoch-check-command.js @@ -0,0 +1,298 @@ +/* eslint-disable no-await-in-loop */ +import Command from '../../command.js'; +import { COMMAND_RETRIES, TRANSACTION_CONFIRMATIONS } from '../../../constants/constants.js'; + +class EpochCheckCommand extends Command { + constructor(ctx) { + super(ctx); + this.commandExecutor = ctx.commandExecutor; + this.repositoryModuleManager = ctx.repositoryModuleManager; + this.networkModuleManager = ctx.networkModuleManager; + this.shardingTableService = ctx.shardingTableService; + this.blockchainModuleManager = ctx.blockchainModuleManager; + this.serviceAgreementService = ctx.serviceAgreementService; + } + + async execute(command) { + await Promise.all( + this.blockchainModuleManager.getImplementationNames().map(async (blockchain) => { + const commitWindowDurationPerc = + await this.blockchainModuleManager.getCommitWindowDurationPerc(blockchain); + const proofWindowDurationPerc = + await this.blockchainModuleManager.getProofWindowDurationPerc(blockchain); + const totalTransactions = await this.calculateTotalTransactions( + blockchain, + commitWindowDurationPerc, + proofWindowDurationPerc, + command.period, + ); + await Promise.all([ + this.scheduleSubmitCommitCommands( + blockchain, + Math.floor(totalTransactions / 2), + commitWindowDurationPerc, + ), + this.scheduleCalculateProofsCommands( + blockchain, + Math.ceil(totalTransactions / 2), + proofWindowDurationPerc, + ), + ]); + }), + ); + return Command.repeat(); + } + + async scheduleSubmitCommitCommands(blockchain, maxTransactions, commitWindowDurationPerc) { + const timestamp = await this.blockchainModuleManager.getBlockchainTimestamp(blockchain); + const eligibleAgreementForSubmitCommit = + await this.repositoryModuleManager.getEligibleAgreementsForSubmitCommit( + timestamp, + blockchain, + commitWindowDurationPerc, + ); + + const scheduleSubmitCommitCommands = []; + const updateServiceAgreementsLastCommitEpoch = []; + for (const serviceAgreement of eligibleAgreementForSubmitCommit) { + if (scheduleSubmitCommitCommands.length >= maxTransactions) break; + + const rank = await this.calculateRank( + blockchain, + serviceAgreement.keyword, + serviceAgreement.hash_function_id, + ); + + const r0 = await this.blockchainModuleManager.getR0(blockchain); + + if (rank < r0) { + this.logger.trace( + `Calculated rank: ${ + rank + 1 + } lower than R0: ${r0}. Scheduling submit commit command for agreement id: ${ + serviceAgreement.agreement_id + }`, + ); + scheduleSubmitCommitCommands.push( + this.scheduleSubmitCommitCommand(serviceAgreement), + ); + } else { + this.logger.trace( + `Calculated rank: ${ + rank + 1 + } higher than R0: ${r0}. Skipping scheduling submit commit command for agreement id: ${ + serviceAgreement.agreement_id + }`, + ); + } + + updateServiceAgreementsLastCommitEpoch.push( + this.repositoryModuleManager.updateServiceAgreementLastCommitEpoch( + serviceAgreement.agreement_id, + serviceAgreement.current_epoch, + ), + ); + } + await Promise.all([ + ...scheduleSubmitCommitCommands, + ...updateServiceAgreementsLastCommitEpoch, + ]); + } + + async scheduleCalculateProofsCommands(blockchain, maxTransactions, proofWindowDurationPerc) { + const timestamp = await this.blockchainModuleManager.getBlockchainTimestamp(blockchain); + const eligibleAgreementsForSubmitProofs = + await this.repositoryModuleManager.getEligibleAgreementsForSubmitProof( + timestamp, + blockchain, + proofWindowDurationPerc, + ); + const scheduleSubmitProofCommands = []; + const updateServiceAgreementsLastProofEpoch = []; + for (const serviceAgreement of eligibleAgreementsForSubmitProofs) { + if (scheduleSubmitProofCommands.length >= maxTransactions) break; + + const eligibleForReward = await this.isEligibleForRewards( + blockchain, + serviceAgreement.agreement_id, + serviceAgreement.current_epoch, + serviceAgreement.state_index, + ); + if (eligibleForReward) { + this.logger.trace( + `Node is eligible for rewards for agreement id: ${serviceAgreement.agreement_id}. Scheduling submit proof command.`, + ); + + scheduleSubmitProofCommands.push( + this.scheduleSubmitProofsCommand(serviceAgreement), + ); + } else { + this.logger.trace( + `Node is not eligible for rewards for agreement id: ${serviceAgreement.agreement_id}. Skipping scheduling submit proof command.`, + ); + } + updateServiceAgreementsLastProofEpoch.push( + this.repositoryModuleManager.updateServiceAgreementLastProofEpoch( + serviceAgreement.agreement_id, + serviceAgreement.current_epoch, + ), + ); + } + await Promise.all([ + ...scheduleSubmitProofCommands, + ...updateServiceAgreementsLastProofEpoch, + ]); + } + + async calculateRank(blockchain, keyword, hashFunctionId) { + const r2 = await this.blockchainModuleManager.getR2(blockchain); + const neighbourhood = await this.shardingTableService.findNeighbourhood( + blockchain, + keyword, + r2, + hashFunctionId, + false, + ); + + const scores = await Promise.all( + neighbourhood.map(async (node) => ({ + score: await this.serviceAgreementService.calculateScore( + node.peer_id, + blockchain, + keyword, + hashFunctionId, + ), + peerId: node.peer_id, + })), + ); + + scores.sort((a, b) => b.score - a.score); + + return scores.findIndex( + (node) => node.peerId === this.networkModuleManager.getPeerId().toB58String(), + ); + } + + async isEligibleForRewards(blockchain, agreementId, epoch, stateIndex) { + const r0 = await this.blockchainModuleManager.getR0(blockchain); + const identityId = await this.blockchainModuleManager.getIdentityId(blockchain); + const commits = await this.blockchainModuleManager.getTopCommitSubmissions( + blockchain, + agreementId, + epoch, + stateIndex, + ); + + for (const commit of commits.slice(0, r0)) { + if (Number(commit.identityId) === identityId && Number(commit.score) !== 0) { + return true; + } + } + + return false; + } + + async scheduleSubmitCommitCommand(agreement) { + const commandData = { + blockchain: agreement.blockchain_id, + contract: agreement.asset_storage_contract_address, + tokenId: agreement.token_id, + keyword: agreement.keyword, + hashFunctionId: agreement.hash_function_id, + epoch: agreement.current_epoch, + agreementId: agreement.agreement_id, + stateIndex: agreement.state_index, + }; + + await this.commandExecutor.add({ + name: 'submitCommitCommand', + sequence: [], + retries: COMMAND_RETRIES.SUBMIT_COMMIT, + data: commandData, + transactional: false, + }); + } + + async scheduleSubmitProofsCommand(agreement) { + const commandData = { + blockchain: agreement.blockchain_id, + contract: agreement.asset_storage_contract_address, + tokenId: agreement.token_id, + keyword: agreement.keyword, + hashFunctionId: agreement.hash_function_id, + epoch: agreement.current_epoch, + agreementId: agreement.agreement_id, + assertionId: agreement.assertion_id, + stateIndex: agreement.state_index, + }; + + return this.commandExecutor.add({ + name: 'submitProofsCommand', + sequence: [], + data: commandData, + retries: COMMAND_RETRIES.SUBMIT_PROOFS, + transactional: false, + }); + } + + async calculateTotalTransactions( + blockchain, + commitWindowDurationPerc, + proofWindowDurationPerc, + commandPeriod, + ) { + const epochLength = await this.blockchainModuleManager.getEpochLength(blockchain); + + const commitWindowDuration = (epochLength * commitWindowDurationPerc) / 100; + const proofWindowDuration = (epochLength * proofWindowDurationPerc) / 100; + + const totalTransactionTime = Math.min(commitWindowDuration, proofWindowDuration); + + const blockTime = this.blockchainModuleManager.getBlockTimeMillis(blockchain) / 1000; + const timePerTransaction = blockTime * TRANSACTION_CONFIRMATIONS; + + const totalTransactions = Math.floor(totalTransactionTime / timePerTransaction); + + const epochChecksInWindow = Math.floor(totalTransactionTime / (commandPeriod / 1000)); + + const transactionsPerEpochCheck = Math.floor(totalTransactions / epochChecksInWindow); + + return transactionsPerEpochCheck; + } + + calculateCommandPeriod() { + const devEnvironment = + process.env.NODE_ENV === 'development' || process.env.NODE_ENV === 'test'; + + return devEnvironment ? 30_000 : 120_000; + } + + /** + * Recover system from failure + * @param command + * @param error + */ + async recover(command, error) { + this.logger.warn(`Failed to execute ${command.name}: error: ${error.message}`); + + return Command.repeat(); + } + + /** + * Builds default epochCheckCommand + * @param map + * @returns {{add, data: *, delay: *, deadline: *}} + */ + default(map) { + const command = { + name: 'epochCheckCommand', + data: {}, + transactional: false, + period: this.calculateCommandPeriod(), + }; + Object.assign(command, map); + return command; + } +} + +export default EpochCheckCommand; diff --git a/src/commands/protocols/common/epoch-command.js b/src/commands/protocols/common/epoch-command.js deleted file mode 100644 index 86950075e..000000000 --- a/src/commands/protocols/common/epoch-command.js +++ /dev/null @@ -1,130 +0,0 @@ -import Command from '../../command.js'; -import { OPERATION_ID_STATUS } from '../../../constants/constants.js'; - -class EpochCommand extends Command { - constructor(ctx) { - super(ctx); - this.commandExecutor = ctx.commandExecutor; - this.blockchainModuleManager = ctx.blockchainModuleManager; - this.operationIdService = ctx.operationIdService; - } - - async scheduleNextEpochCheck( - blockchain, - agreementId, - contract, - tokenId, - keyword, - hashFunctionId, - agreementData, - operationId, - assertionId, - ) { - const currentEpoch = await this.calculateCurrentEpoch( - Number(agreementData.startTime), - Number(agreementData.epochLength), - blockchain, - ); - const nextEpochStartTime = - Number(agreementData.startTime) + - Number(agreementData.epochLength) * (currentEpoch + 1); - - const commitWindowDurationPerc = - await this.blockchainModuleManager.getCommitWindowDurationPerc(blockchain); - // delay by 10% of commit window length - const offset = ((Number(agreementData.epochLength) * commitWindowDurationPerc) / 100) * 0.1; - - const now = await this.blockchainModuleManager.getBlockchainTimestamp(blockchain); - - const delay = nextEpochStartTime - now + offset; - - this.logger.trace( - `Scheduling next epoch check for agreement id: ${agreementId} in ${delay} seconds.`, - ); - await this.commandExecutor.add({ - name: 'epochCheckCommand', - sequence: [], - delay: delay * 1000, - data: { - blockchain, - agreementId, - contract, - tokenId, - keyword, - hashFunctionId, - operationId, - assertionId, - }, - transactional: false, - }); - } - - async handleExpiredAsset(agreementId, operationId, epoch) { - this.logger.trace( - `Asset lifetime for agreement id: ${agreementId} has expired. Operation id: ${operationId}`, - ); - this.operationIdService.emitChangeEvent( - OPERATION_ID_STATUS.COMMIT_PROOF.EPOCH_CHECK_END, - operationId, - agreementId, - epoch, - ); - } - - async calculateCurrentEpoch(startTime, epochLength, blockchain) { - const now = await this.blockchainModuleManager.getBlockchainTimestamp(blockchain); - return Math.floor((Number(now) - Number(startTime)) / Number(epochLength)); - } - - /** - * Recover system from failure - * @param command - * @param error - */ - async recover(command, error) { - this.logger.warn(`Failed to execute ${command.name}: error: ${error.message}`); - - this.operationIdService.emitChangeEvent( - OPERATION_ID_STATUS.FAILED, - command.data.operationId, - error.message, - this.errorType, - command.data.epoch, - ); - - await this.scheduleNextEpochCheck( - command.data.blockchain, - command.data.agreementId, - command.data.contract, - command.data.tokenId, - command.data.keyword, - command.data.hashFunctionId, - command.data.agreementData, - command.data.operationId, - command.data.assertionId, - ); - - return Command.empty(); - } - - async retryFinished(command) { - this.recover(command, `Max retry count for command: ${command.name} reached!`); - } - - /** - * Builds default epochCommand - * @param map - * @returns {{add, data: *, delay: *, deadline: *}} - */ - default(map) { - const command = { - name: 'epochCommand', - delay: 0, - transactional: false, - }; - Object.assign(command, map); - return command; - } -} - -export default EpochCommand; diff --git a/src/commands/protocols/common/submit-commit-command.js b/src/commands/protocols/common/submit-commit-command.js new file mode 100644 index 000000000..fc74f34b4 --- /dev/null +++ b/src/commands/protocols/common/submit-commit-command.js @@ -0,0 +1,149 @@ +import { v4 as uuidv4 } from 'uuid'; +import { OPERATION_ID_STATUS, ERROR_TYPE, COMMAND_RETRIES } from '../../../constants/constants.js'; +import Command from '../../command.js'; + +class SubmitCommitCommand extends Command { + constructor(ctx) { + super(ctx); + this.commandExecutor = ctx.commandExecutor; + this.blockchainModuleManager = ctx.blockchainModuleManager; + this.operationIdService = ctx.operationIdService; + + this.errorType = ERROR_TYPE.COMMIT_PROOF.SUBMIT_COMMIT_ERROR; + } + + async execute(command) { + const { + blockchain, + contract, + tokenId, + keyword, + hashFunctionId, + epoch, + agreementId, + stateIndex, + } = command.data; + + // TODO: review operationId + const operationId = uuidv4(); + + if (command.retries === COMMAND_RETRIES.SUBMIT_COMMIT) { + this.operationIdService.emitChangeEvent( + OPERATION_ID_STATUS.COMMIT_PROOF.SUBMIT_COMMIT_START, + operationId, + agreementId, + epoch, + ); + } + + this.logger.trace( + `Started ${command.name} for agreement id: ${agreementId} ` + + `contract: ${contract}, token id: ${tokenId}, keyword: ${keyword}, ` + + `hash function id: ${hashFunctionId}, epoch: ${epoch}, stateIndex: ${stateIndex}. Retry number ${ + COMMAND_RETRIES.SUBMIT_COMMIT - command.retries + 1 + }`, + ); + + if (command.retries !== COMMAND_RETRIES.SUBMIT_COMMIT) { + const alreadySubmitted = await this.commitAlreadySubmitted( + blockchain, + agreementId, + epoch, + stateIndex, + ); + if (alreadySubmitted) { + this.logger.trace( + `Commit already submitted for blockchain: ${blockchain} agreement id: ${agreementId}, epoch: ${epoch}, state index: ${stateIndex}`, + ); + return Command.empty(); + } + } + + await this.blockchainModuleManager.submitCommit( + blockchain, + contract, + tokenId, + keyword, + hashFunctionId, + epoch, + stateIndex, + async (result) => { + if (!result.error) { + this.logger.trace( + `Successfully executed ${command.name} for agreement id: ${agreementId} ` + + `contract: ${contract}, token id: ${tokenId}, keyword: ${keyword}, ` + + `hash function id: ${hashFunctionId}. Retry number ${ + COMMAND_RETRIES.SUBMIT_COMMIT - command.retries + 1 + }`, + ); + this.operationIdService.emitChangeEvent( + OPERATION_ID_STATUS.COMMIT_PROOF.SUBMIT_COMMIT_END, + operationId, + agreementId, + epoch, + ); + } else if (command.retries - 1 === 0) { + const errorMessage = `Failed executing submit commit command, maximum number of retries reached. Error: ${result.error.message}.`; + this.logger.error(errorMessage); + this.operationIdService.emitChangeEvent( + OPERATION_ID_STATUS.FAILED, + operationId, + errorMessage, + this.errorType, + epoch, + ); + } else { + const blockTime = this.blockchainModuleManager.getBlockTimeMillis(blockchain); + this.logger.warn( + `Failed executing submit commit command, retrying in ${blockTime}ms. Error: ${result.error.message}`, + ); + await this.commandExecutor.add({ + name: 'submitCommitCommand', + sequence: [], + delay: blockTime, + retries: command.retries - 1, + data: command.data, + transactional: false, + }); + } + }, + ); + + return Command.empty(); + } + + async commitAlreadySubmitted(blockchain, agreementId, epoch, stateIndex) { + const commits = await this.blockchainModuleManager.getTopCommitSubmissions( + blockchain, + agreementId, + epoch, + stateIndex, + ); + const identityId = await this.blockchainModuleManager.getIdentityId(blockchain); + + for (const commit of commits) { + if (Number(commit.identityId) === identityId) { + return true; + } + } + + return false; + } + + /** + * Builds default handleStoreInitCommand + * @param map + * @returns {{add, data: *, delay: *, deadline: *}} + */ + default(map) { + const command = { + name: 'submitCommitCommand', + delay: 0, + transactional: false, + }; + Object.assign(command, map); + return command; + } +} + +export default SubmitCommitCommand; diff --git a/src/commands/protocols/publish/receiver/submit-proofs-command.js b/src/commands/protocols/common/submit-proofs-command.js similarity index 57% rename from src/commands/protocols/publish/receiver/submit-proofs-command.js rename to src/commands/protocols/common/submit-proofs-command.js index 817bd1415..8ac922ff2 100644 --- a/src/commands/protocols/publish/receiver/submit-proofs-command.js +++ b/src/commands/protocols/common/submit-proofs-command.js @@ -1,16 +1,20 @@ -import EpochCommand from '../../common/epoch-command.js'; +import { v4 as uuidv4 } from 'uuid'; import { OPERATION_ID_STATUS, ERROR_TYPE, COMMAND_RETRIES, - BLOCK_TIME, -} from '../../../../constants/constants.js'; + TRIPLE_STORE_REPOSITORIES, +} from '../../../constants/constants.js'; +import Command from '../../command.js'; -class SubmitProofsCommand extends EpochCommand { +class SubmitProofsCommand extends Command { constructor(ctx) { super(ctx); this.blockchainModuleManager = ctx.blockchainModuleManager; + this.repositoryModuleManager = ctx.repositoryModuleManager; + this.validationModuleManager = ctx.validationModuleManager; + this.tripleStoreService = ctx.tripleStoreService; this.operationIdService = ctx.operationIdService; this.repositoryModuleManager = ctx.repositoryModuleManager; @@ -20,21 +24,19 @@ class SubmitProofsCommand extends EpochCommand { async execute(command) { const { blockchain, - leaf, - proof, - agreementData, - epoch, - agreementId, contract, tokenId, keyword, hashFunctionId, - operationId, - identityId, + epoch, + agreementId, assertionId, stateIndex, } = command.data; + // TODO: review operationId + const operationId = uuidv4(); + this.logger.trace( `Started ${command.name} for agreement id: ${agreementId} ` + `contract: ${contract}, token id: ${tokenId}, keyword: ${keyword}, ` + @@ -43,39 +45,68 @@ class SubmitProofsCommand extends EpochCommand { }`, ); - const commits = await this.blockchainModuleManager.getTopCommitSubmissions( + if (command.retries === COMMAND_RETRIES.SUBMIT_PROOFS) { + this.operationIdService.emitChangeEvent( + OPERATION_ID_STATUS.COMMIT_PROOF.CALCULATE_PROOFS_START, + operationId, + agreementId, + epoch, + ); + } + + this.logger.trace(`Calculating proofs for agreement id : ${agreementId}`); + const { challenge } = await this.blockchainModuleManager.getChallenge( blockchain, - agreementId, + contract, + tokenId, epoch, stateIndex, ); - if (this.proofAlreadySubmitted(commits, identityId)) { - this.logger.trace( - `Proofs already submitted for agreement id: ${agreementId} and epoch: ${epoch}`, - ); - await this.scheduleNextEpochCheck( - blockchain, - agreementId, - contract, - tokenId, - keyword, - hashFunctionId, - agreementData, - operationId, - assertionId, - ); - return EpochCommand.empty(); + + const assertion = await this.tripleStoreService.getAssertion( + TRIPLE_STORE_REPOSITORIES.PUBLIC_CURRENT, + assertionId, + ); + + if (!assertion.length) { + this.logger.trace(`Assertion with id: ${assertionId} not found in triple store.`); + return Command.empty(); } + + const { leaf, proof } = this.validationModuleManager.getMerkleProof( + assertion, + Number(challenge), + ); + if (command.retries === COMMAND_RETRIES.SUBMIT_PROOFS) { + this.operationIdService.emitChangeEvent( + OPERATION_ID_STATUS.COMMIT_PROOF.CALCULATE_PROOFS_END, + operationId, + agreementId, + epoch, + ); + this.operationIdService.emitChangeEvent( OPERATION_ID_STATUS.COMMIT_PROOF.SUBMIT_PROOFS_START, operationId, agreementId, epoch, ); + } else { + const alreadySubmitted = await this.proofAlreadySubmitted( + blockchain, + agreementId, + epoch, + stateIndex, + ); + if (alreadySubmitted) { + this.logger.trace( + `Proofs already submitted for blockchain: ${blockchain} agreement id: ${agreementId}, epoch: ${epoch}, state index: ${stateIndex}`, + ); + return Command.empty(); + } } - const that = this; await this.blockchainModuleManager.sendProof( blockchain, contract, @@ -88,7 +119,7 @@ class SubmitProofsCommand extends EpochCommand { stateIndex, async (result) => { if (!result.error) { - that.logger.trace( + this.logger.trace( `Successfully executed ${command.name} for agreement id: ${agreementId} ` + `contract: ${contract}, token id: ${tokenId}, keyword: ${keyword}, ` + `hash function id: ${hashFunctionId}. Retry number ${ @@ -96,7 +127,7 @@ class SubmitProofsCommand extends EpochCommand { }`, ); - that.operationIdService.emitChangeEvent( + this.operationIdService.emitChangeEvent( OPERATION_ID_STATUS.COMMIT_PROOF.SUBMIT_PROOFS_END, operationId, agreementId, @@ -104,51 +135,48 @@ class SubmitProofsCommand extends EpochCommand { ); } else if (command.retries - 1 === 0) { const errorMessage = `Failed executing submit proofs command, maximum number of retries reached. Error: ${result.error.message}. Scheduling next epoch check.`; - that.logger.error(errorMessage); - that.operationIdService.emitChangeEvent( + this.logger.error(errorMessage); + this.operationIdService.emitChangeEvent( OPERATION_ID_STATUS.FAILED, operationId, errorMessage, - that.errorType, + this.errorType, epoch, ); } else { - const commandDelay = BLOCK_TIME * 1000; // one block - that.logger.warn( - `Failed executing submit proofs command, retrying in ${commandDelay}ms. Error: ${result.error.message}`, + const blockTime = this.blockchainModuleManager.getBlockTimeMillis(blockchain); + this.logger.warn( + `Failed executing submit proofs command, retrying in ${blockTime}ms. Error: ${result.error.message}`, ); - await that.commandExecutor.add({ + await this.commandExecutor.add({ name: 'submitProofsCommand', sequence: [], - delay: commandDelay, + delay: blockTime, data: command.data, retries: command.retries - 1, transactional: false, }); - return; } - await that.scheduleNextEpochCheck( - blockchain, - agreementId, - contract, - tokenId, - keyword, - hashFunctionId, - agreementData, - operationId, - assertionId, - ); }, ); - return EpochCommand.empty(); + return Command.empty(); } - proofAlreadySubmitted(commits, myIdentity) { - commits.forEach((commit) => { - if (Number(commit.identityId) === myIdentity && Number(commit.score) === 0) { + async proofAlreadySubmitted(blockchain, agreementId, epoch, stateIndex) { + const commits = await this.blockchainModuleManager.getTopCommitSubmissions( + blockchain, + agreementId, + epoch, + stateIndex, + ); + const identityId = await this.blockchainModuleManager.getIdentityId(blockchain); + + for (const commit of commits) { + if (Number(commit.identityId) === identityId && Number(commit.score) === 0) { return true; } - }); + } + return false; } diff --git a/src/commands/protocols/publish/receiver/calculate-proofs-command.js b/src/commands/protocols/publish/receiver/calculate-proofs-command.js deleted file mode 100644 index 04a342e8e..000000000 --- a/src/commands/protocols/publish/receiver/calculate-proofs-command.js +++ /dev/null @@ -1,171 +0,0 @@ -import EpochCommand from '../../common/epoch-command.js'; -import { - OPERATION_ID_STATUS, - ERROR_TYPE, - COMMAND_RETRIES, - TRIPLE_STORE_REPOSITORIES, -} from '../../../../constants/constants.js'; - -class CalculateProofsCommand extends EpochCommand { - constructor(ctx) { - super(ctx); - this.commandExecutor = ctx.commandExecutor; - this.validationModuleManager = ctx.validationModuleManager; - this.blockchainModuleManager = ctx.blockchainModuleManager; - this.tripleStoreService = ctx.tripleStoreService; - this.operationIdService = ctx.operationIdService; - this.dataService = ctx.dataService; - this.errorType = ERROR_TYPE.COMMIT_PROOF.CALCULATE_PROOFS_ERROR; - } - - async execute(command) { - const { - blockchain, - contract, - tokenId, - keyword, - hashFunctionId, - agreementData, - agreementId, - identityId, - operationId, - } = command.data; - const assertionIds = await this.blockchainModuleManager.getAssertionIds( - blockchain, - contract, - tokenId, - ); - const stateIndex = assertionIds.length - 1; - const assertionId = assertionIds[stateIndex]; - - this.logger.trace( - `Started ${command.name} for agreement id: ${agreementId} ` + - `blockchain:${blockchain}, contract: ${contract}, token id: ${tokenId}, ` + - `keyword: ${keyword}, hash function id: ${hashFunctionId} and stateIndex: ${stateIndex}`, - ); - - const epoch = await this.calculateCurrentEpoch( - Number(agreementData.startTime), - Number(agreementData.epochLength), - blockchain, - ); - - this.operationIdService.emitChangeEvent( - OPERATION_ID_STATUS.COMMIT_PROOF.CALCULATE_PROOFS_START, - operationId, - agreementId, - epoch, - ); - - if ( - !(await this.isEligibleForRewards( - blockchain, - agreementId, - epoch, - identityId, - stateIndex, - )) - ) { - await this.scheduleNextEpochCheck( - blockchain, - agreementId, - contract, - tokenId, - keyword, - hashFunctionId, - agreementData, - operationId, - assertionId, - ); - - return EpochCommand.empty(); - } - - this.logger.trace(`Calculating proofs for agreement id : ${agreementId}`); - const { challenge } = await this.blockchainModuleManager.getChallenge( - blockchain, - contract, - tokenId, - epoch, - stateIndex, - ); - - const assertion = await this.tripleStoreService.getAssertion( - TRIPLE_STORE_REPOSITORIES.PUBLIC_CURRENT, - assertionId, - ); - - if (!assertion.length) { - this.logger.trace( - `Assertion with id: ${assertionId} not found in triple store. Not scheduling next epcoh checks.`, - ); - return EpochCommand.empty(); - } - - const { leaf, proof } = this.validationModuleManager.getMerkleProof( - assertion, - Number(challenge), - ); - - await this.commandExecutor.add({ - name: 'submitProofsCommand', - sequence: [], - delay: 0, - data: { - ...command.data, - leaf, - proof, - stateIndex, - }, - retries: COMMAND_RETRIES.SUBMIT_PROOFS, - transactional: false, - }); - - this.operationIdService.emitChangeEvent( - OPERATION_ID_STATUS.COMMIT_PROOF.CALCULATE_PROOFS_END, - operationId, - agreementId, - epoch, - ); - return EpochCommand.empty(); - } - - async isEligibleForRewards(blockchain, agreementId, epoch, identityId, stateIndex) { - const r0 = await this.blockchainModuleManager.getR0(blockchain); - - const commits = await this.blockchainModuleManager.getTopCommitSubmissions( - blockchain, - agreementId, - epoch, - stateIndex, - ); - for (let i = 0; i < Math.min(r0, commits.length); i += 1) { - if (Number(commits[i].identityId) === identityId) { - this.logger.trace(`Node is eligible for rewards for agreement id: ${agreementId}`); - - return true; - } - } - - this.logger.trace(`Node is not eligible for rewards for agreement id: ${agreementId}`); - - return false; - } - - /** - * Builds default calculateProofsCommand - * @param map - * @returns {{add, data: *, delay: *, deadline: *}} - */ - default(map) { - const command = { - name: 'calculateProofsCommand', - delay: 0, - transactional: false, - }; - Object.assign(command, map); - return command; - } -} - -export default CalculateProofsCommand; diff --git a/src/commands/protocols/publish/receiver/epoch-check-command.js b/src/commands/protocols/publish/receiver/epoch-check-command.js deleted file mode 100644 index a1471ba64..000000000 --- a/src/commands/protocols/publish/receiver/epoch-check-command.js +++ /dev/null @@ -1,143 +0,0 @@ -import EpochCommand from '../../common/epoch-command.js'; -import { - OPERATION_ID_STATUS, - ERROR_TYPE, - COMMAND_RETRIES, - TRIPLE_STORE_REPOSITORIES, -} from '../../../../constants/constants.js'; - -class EpochCheckCommand extends EpochCommand { - constructor(ctx) { - super(ctx); - this.commandExecutor = ctx.commandExecutor; - this.blockchainModuleManager = ctx.blockchainModuleManager; - this.repositoryModuleManager = ctx.repositoryModuleManager; - this.serviceAgreementService = ctx.serviceAgreementService; - this.operationIdService = ctx.operationIdService; - this.tripleStoreService = ctx.tripleStoreService; - - this.errorType = ERROR_TYPE.COMMIT_PROOF.EPOCH_CHECK_ERROR; - } - - async execute(command) { - const { blockchain, agreementId, contract, tokenId, keyword, hashFunctionId, operationId } = - command.data; - - this.logger.trace( - `Started ${command.name} for agreement id: ${agreementId} ` + - `contract: ${contract}, token id: ${tokenId}, keyword: ${keyword}, ` + - `hash function id: ${hashFunctionId}`, - ); - - const agreementData = await this.blockchainModuleManager.getAgreementData( - blockchain, - agreementId, - ); - const epoch = await this.calculateCurrentEpoch( - Number(agreementData.startTime), - Number(agreementData.epochLength), - blockchain, - ); - this.logger.trace(`Epoch number: ${epoch}`); - this.operationIdService.emitChangeEvent( - OPERATION_ID_STATUS.COMMIT_PROOF.EPOCH_CHECK_START, - operationId, - agreementId, - epoch, - ); - const assertionIds = await this.blockchainModuleManager.getAssertionIds( - blockchain, - contract, - tokenId, - ); - const stateIndex = assertionIds.length - 1; - const assertionId = assertionIds[stateIndex]; - - if (this.assetLifetimeExpired(agreementData, epoch)) { - await this.handleExpiredAsset(agreementId, operationId, epoch); - return EpochCommand.empty(); - } - const assertionExists = await this.tripleStoreService.assertionExists( - TRIPLE_STORE_REPOSITORIES.PUBLIC_CURRENT, - assertionId, - ); - - if (!assertionExists) { - this.logger.trace( - `Assertion with id: ${assertionId} not found in triple store. Not scheduling next epoch checks.`, - ); - return EpochCommand.empty(); - } - - const commitWindowOpen = await this.blockchainModuleManager.isCommitWindowOpen( - blockchain, - agreementId, - epoch, - stateIndex, - ); - if (!commitWindowOpen) { - this.logger.trace( - `Commit window for agreement id: ${agreementId} is closed. Scheduling next epoch check.`, - ); - await this.scheduleNextEpochCheck( - blockchain, - agreementId, - contract, - tokenId, - keyword, - hashFunctionId, - agreementData, - operationId, - assertionId, - ); - return this.finishEpochCheckCommand(operationId, agreementId, epoch); - } - - const identityId = - command.data.identityId ?? - (await this.blockchainModuleManager.getIdentityId(blockchain)); - - await this.commandExecutor.add({ - name: 'submitCommitCommand', - sequence: [], - delay: 0, - period: 12 * 1000, // todo: get from blockchain / oracle - retries: COMMAND_RETRIES.SUBMIT_COMMIT, - data: { ...command.data, agreementData, identityId, epoch, stateIndex }, - transactional: false, - }); - - return this.finishEpochCheckCommand(operationId, agreementId, epoch); - } - - finishEpochCheckCommand(operationId, agreementId, epoch) { - this.operationIdService.emitChangeEvent( - OPERATION_ID_STATUS.COMMIT_PROOF.EPOCH_CHECK_END, - operationId, - agreementId, - epoch, - ); - return EpochCommand.empty(); - } - - assetLifetimeExpired(agreementData, epoch) { - return epoch >= Number(agreementData.epochsNumber); - } - - /** - * Builds default handleStoreInitCommand - * @param map - * @returns {{add, data: *, delay: *, deadline: *}} - */ - default(map) { - const command = { - name: 'epochCheckCommand', - delay: 0, - transactional: false, - }; - Object.assign(command, map); - return command; - } -} - -export default EpochCheckCommand; diff --git a/src/commands/protocols/publish/receiver/submit-commit-command.js b/src/commands/protocols/publish/receiver/submit-commit-command.js deleted file mode 100644 index 5e436e9ce..000000000 --- a/src/commands/protocols/publish/receiver/submit-commit-command.js +++ /dev/null @@ -1,263 +0,0 @@ -import EpochCommand from '../../common/epoch-command.js'; -import { - OPERATION_ID_STATUS, - ERROR_TYPE, - COMMAND_RETRIES, - BLOCK_TIME, -} from '../../../../constants/constants.js'; - -class SubmitCommitCommand extends EpochCommand { - constructor(ctx) { - super(ctx); - this.commandExecutor = ctx.commandExecutor; - this.blockchainModuleManager = ctx.blockchainModuleManager; - this.serviceAgreementService = ctx.serviceAgreementService; - this.operationIdService = ctx.operationIdService; - this.shardingTableService = ctx.shardingTableService; - this.networkModuleManager = ctx.networkModuleManager; - this.repositoryModuleManager = ctx.repositoryModuleManager; - - this.errorType = ERROR_TYPE.COMMIT_PROOF.SUBMIT_COMMIT_ERROR; - } - - async execute(command) { - const { - blockchain, - contract, - tokenId, - keyword, - hashFunctionId, - epoch, - agreementData, - agreementId, - identityId, - operationId, - assertionId, - stateIndex, - } = command.data; - if (command.retries === COMMAND_RETRIES.SUBMIT_COMMIT) { - this.operationIdService.emitChangeEvent( - OPERATION_ID_STATUS.COMMIT_PROOF.SUBMIT_COMMIT_START, - operationId, - agreementId, - epoch, - ); - } - - this.logger.trace( - `Started ${command.name} for agreement id: ${agreementId} ` + - `contract: ${contract}, token id: ${tokenId}, keyword: ${keyword}, ` + - `hash function id: ${hashFunctionId}, epoch: ${epoch}, stateIndex: ${stateIndex}. Retry number ${ - COMMAND_RETRIES.SUBMIT_COMMIT - command.retries + 1 - }`, - ); - - const commits = await this.blockchainModuleManager.getTopCommitSubmissions( - blockchain, - agreementId, - epoch, - stateIndex, - ); - - this.logger.trace('Commit submissions:'); - this.logger.trace(JSON.stringify(commits, null, 1)); - - // calculate proofs -> schedule proof submission -> schedule next epoch - if (this.alreadyCommitted(commits, identityId)) { - // How this even possible? - this.logger.trace( - `Current epoch's commit has already been submitted for agreement id: ${agreementId}.`, - ); - await this.commandExecutor.add({ - name: 'calculateProofsCommand', - sequence: [], - delay: 0, // We should calculate proofs after commit phase end + only for winning nodes. - data: { ...command.data, agreementData, identityId }, - transactional: false, - }); - return EpochCommand.empty(); - } - - this.logger.trace( - `Calculating commit submission score for agreement id: ${agreementId}...`, - ); - - // submit commit -> calculate proofs -> schedule proof submission -> schedule next epoch - const rank = await this.calculateRank(blockchain, keyword, hashFunctionId); - - const r0 = await this.blockchainModuleManager.getR0(blockchain); - - if (rank >= r0) { - this.logger.trace( - `Calculated rank: ${rank} higher than R0: ${r0}. Scheduling next epoch check for agreement id: ${agreementId}`, - ); - await this.scheduleNextEpochCheck( - blockchain, - agreementId, - contract, - tokenId, - keyword, - hashFunctionId, - agreementData, - operationId, - assertionId, - ); - return EpochCommand.empty(); - } - - const that = this; - await this.blockchainModuleManager.submitCommit( - blockchain, - contract, - tokenId, - keyword, - hashFunctionId, - epoch, - stateIndex, - async (result) => { - if (!result.error) { - that.logger.trace( - `Successfully executed ${command.name} for agreement id: ${agreementId} ` + - `contract: ${contract}, token id: ${tokenId}, keyword: ${keyword}, ` + - `hash function id: ${hashFunctionId}. Retry number ${ - COMMAND_RETRIES.SUBMIT_COMMIT - command.retries + 1 - }`, - ); - - const currentEpochStartTime = - Number(agreementData.startTime) + Number(agreementData.epochLength) * epoch; - - const proofWindowDurationPerc = - await that.blockchainModuleManager.getProofWindowDurationPerc(blockchain); - - const proofWindowDuration = - (Number(agreementData.epochLength) * proofWindowDurationPerc) / 100; - - const proofWindowStartTime = - currentEpochStartTime + - Math.floor( - (Number(agreementData.epochLength) * - Number(agreementData.proofWindowOffsetPerc)) / - 100, - ); - // we are not using Date.now() here becouse we have an issue with hardhat blockchain time - const timeNow = await that.blockchainModuleManager.getBlockchainTimestamp(); - const delay = - that.serviceAgreementService.randomIntFromInterval( - proofWindowStartTime + 0.1 * proofWindowDuration, - proofWindowStartTime + proofWindowDuration - 0.1 * proofWindowDuration, - ) - timeNow; - - that.logger.trace( - `Scheduling calculateProofsCommand for agreement id: ${agreementId} in ${delay} seconds`, - ); - - await that.operationIdService.emitChangeEvent( - OPERATION_ID_STATUS.COMMIT_PROOF.SUBMIT_COMMIT_END, - operationId, - agreementId, - epoch, - ); - - await that.commandExecutor.add({ - name: 'calculateProofsCommand', - delay: delay * 1000, - data: { ...command.data, proofWindowStartTime }, - transactional: false, - }); - } else if (command.retries - 1 === 0) { - await that.scheduleNextEpochCheck( - blockchain, - agreementId, - contract, - tokenId, - keyword, - hashFunctionId, - agreementData, - operationId, - assertionId, - ); - const errorMessage = `Failed executing submit commit command, maximum number of retries reached. Error: ${result.error.message}. Scheduling next epoch check.`; - that.logger.error(errorMessage); - await that.operationIdService.emitChangeEvent( - OPERATION_ID_STATUS.FAILED, - operationId, - errorMessage, - that.errorType, - epoch, - ); - } else { - const commandDelay = BLOCK_TIME * 1000; // one block - that.logger.warn( - `Failed executing submit commit command, retrying in ${commandDelay}ms. Error: ${result.error.message}`, - ); - await that.commandExecutor.add({ - name: 'submitCommitCommand', - sequence: [], - delay: commandDelay, - retries: command.retries - 1, - data: command.data, - transactional: false, - }); - } - }, - ); - - return EpochCommand.empty(); - } - - async calculateRank(blockchain, keyword, hashFunctionId) { - const r2 = await this.blockchainModuleManager.getR2(blockchain); - const neighbourhood = await this.shardingTableService.findNeighbourhood( - blockchain, - keyword, - r2, - hashFunctionId, - false, - ); - - const scores = await Promise.all( - neighbourhood.map(async (node) => ({ - score: await this.serviceAgreementService.calculateScore( - node.peer_id, - blockchain, - keyword, - hashFunctionId, - ), - peerId: node.peer_id, - })), - ); - - scores.sort((a, b) => b.score - a.score); - - return scores.findIndex( - (node) => node.peerId === this.networkModuleManager.getPeerId().toB58String(), - ); - } - - alreadyCommitted(commits, myIdentity) { - commits.forEach((commit) => { - if (Number(commit.identityId) === myIdentity) { - return true; - } - }); - return false; - } - - /** - * Builds default handleStoreInitCommand - * @param map - * @returns {{add, data: *, delay: *, deadline: *}} - */ - default(map) { - const command = { - name: 'submitCommitCommand', - delay: 0, - transactional: false, - }; - Object.assign(command, map); - return command; - } -} - -export default SubmitCommitCommand; diff --git a/src/commands/protocols/publish/receiver/v1.0.0/v1-0-0-handle-store-request-command.js b/src/commands/protocols/publish/receiver/v1.0.0/v1-0-0-handle-store-request-command.js index e54767f90..0caa031c6 100644 --- a/src/commands/protocols/publish/receiver/v1.0.0/v1-0-0-handle-store-request-command.js +++ b/src/commands/protocols/publish/receiver/v1.0.0/v1-0-0-handle-store-request-command.js @@ -13,7 +13,6 @@ class HandleStoreRequestCommand extends HandleProtocolMessageCommand { this.validationService = ctx.validationService; this.operationService = ctx.publishService; this.serviceAgreementService = ctx.serviceAgreementService; - this.commandExecutor = ctx.commandExecutor; this.repositoryModuleManager = ctx.repositoryModuleManager; this.blockchainModuleManager = ctx.blockchainModuleManager; this.tripleStoreService = ctx.tripleStoreService; @@ -58,21 +57,30 @@ class HandleStoreRequestCommand extends HandleProtocolMessageCommand { OPERATION_ID_STATUS.PUBLISH.PUBLISH_LOCAL_STORE_START, ); - const assetExists = await this.tripleStoreService.assetExists( + await this.tripleStoreService.localStoreAsset( TRIPLE_STORE_REPOSITORIES.PUBLIC_CURRENT, + assertionId, + assertion, blockchain, contract, tokenId, + keyword, ); - await this.tripleStoreService.localStoreAsset( - TRIPLE_STORE_REPOSITORIES.PUBLIC_CURRENT, - assertionId, - assertion, + await this.repositoryModuleManager.updateServiceAgreementRecord( blockchain, contract, tokenId, + agreementId, + agreementData.startTime, + agreementData.epochsNumber, + agreementData.epochLength, + agreementData.scoreFunctionId, + agreementData.proofWindowOffsetPerc, + hashFunctionId, keyword, + assertionId, + stateIndex, ); await this.repositoryModuleManager.updateServiceAgreementRecord( @@ -96,34 +104,6 @@ class HandleStoreRequestCommand extends HandleProtocolMessageCommand { OPERATION_ID_STATUS.PUBLISH.PUBLISH_LOCAL_STORE_END, ); - const ual = this.ualService.deriveUAL(blockchain, contract, tokenId); - if (!assetExists) { - this.logger.trace( - `Asset with ${ual} not previously present in triple store. Scheduling epoch check command.`, - ); - await this.commandExecutor.add({ - name: 'epochCheckCommand', - sequence: [], - delay: 0, - data: { - blockchain, - contract, - tokenId, - keyword, - hashFunctionId, - operationId, - agreementId, - agreementData, - assertionId, - }, - transactional: false, - }); - } else { - this.logger.trace( - `Asset with ${ual} previously present in triple store. Not scheduling epoch check command.`, - ); - } - return { messageType: NETWORK_MESSAGE_TYPES.RESPONSES.ACK, messageData: {} }; } diff --git a/src/commands/protocols/update/receiver/submit-update-commit-command.js b/src/commands/protocols/update/receiver/submit-update-commit-command.js index 3997d0798..2b36c1d30 100644 --- a/src/commands/protocols/update/receiver/submit-update-commit-command.js +++ b/src/commands/protocols/update/receiver/submit-update-commit-command.js @@ -1,14 +1,14 @@ -import EpochCommand from '../../common/epoch-command.js'; +import Command from '../../../command.js'; import { OPERATION_ID_STATUS, ERROR_TYPE, COMMAND_RETRIES, - BLOCK_TIME, } from '../../../../constants/constants.js'; -class SubmitUpdateCommitCommand extends EpochCommand { +class SubmitUpdateCommitCommand extends Command { constructor(ctx) { super(ctx); + this.commandExecutor = ctx.commandExecutor; this.blockchainModuleManager = ctx.blockchainModuleManager; this.serviceAgreementService = ctx.serviceAgreementService; this.operationIdService = ctx.operationIdService; @@ -60,10 +60,9 @@ class SubmitUpdateCommitCommand extends EpochCommand { if (!hasPendingUpdate) { this.logger.trace(`Not submitting as state is already finalized for update.`); - return EpochCommand.empty(); + return Command.empty(); } - const that = this; await this.blockchainModuleManager.submitUpdateCommit( blockchain, contract, @@ -73,31 +72,31 @@ class SubmitUpdateCommitCommand extends EpochCommand { epoch, async (result) => { if (!result.error) { - that.operationIdService.emitChangeEvent( + this.operationIdService.emitChangeEvent( OPERATION_ID_STATUS.COMMIT_PROOF.SUBMIT_UPDATE_COMMIT_END, operationId, agreementId, epoch, ); - that.logger.info('Successfully executed submit update commit'); + this.logger.info('Successfully executed submit update commit'); } else if (command.retries - 1 === 0) { const errorMessage = `Failed executing submit update commit command, maximum number of retries reached. Error: ${result.error.message}`; - that.logger.error(errorMessage); - that.operationIdService.emitChangeEvent( + this.logger.error(errorMessage); + this.operationIdService.emitChangeEvent( OPERATION_ID_STATUS.FAILED, operationId, errorMessage, - that.errorType, + this.errorType, epoch, ); } else { - const commandDelay = BLOCK_TIME * 1000; // one block - that.logger.warn( - `Failed executing submit update commit command, retrying in ${commandDelay}ms. Error: ${result.error.message}`, + const blockTime = this.blockchainModuleManager.getBlockTimeMillis(blockchain); + this.logger.warn( + `Failed executing submit update commit command, retrying in ${blockTime}ms. Error: ${result.error.message}`, ); - that.commandExecutor.add({ + await this.commandExecutor.add({ name: 'submitUpdateCommitCommand', - delay: commandDelay, + delay: blockTime, retries: command.retries - 1, data: command.data, transactional: false, @@ -106,7 +105,16 @@ class SubmitUpdateCommitCommand extends EpochCommand { }, ); - return EpochCommand.empty(); + return Command.empty(); + } + + async calculateCurrentEpoch(startTime, epochLength, blockchain) { + const now = await this.blockchainModuleManager.getBlockchainTimestamp(blockchain); + return Math.floor((Number(now) - Number(startTime)) / Number(epochLength)); + } + + async retryFinished(command) { + this.recover(command, `Max retry count for command: ${command.name} reached!`); } /** diff --git a/src/commands/protocols/update/receiver/v1.0.0/v1-0-0-handle-update-request-command.js b/src/commands/protocols/update/receiver/v1.0.0/v1-0-0-handle-update-request-command.js index 516d173e8..717a4d999 100644 --- a/src/commands/protocols/update/receiver/v1.0.0/v1-0-0-handle-update-request-command.js +++ b/src/commands/protocols/update/receiver/v1.0.0/v1-0-0-handle-update-request-command.js @@ -7,7 +7,6 @@ import { COMMAND_RETRIES, PENDING_STORAGE_REPOSITORIES, COMMIT_BLOCK_DURATION_IN_BLOCKS, - BLOCK_TIME, COMMITS_DELAY_BETWEEN_NODES_IN_BLOCKS, } from '../../../../../constants/constants.js'; @@ -70,6 +69,7 @@ class HandleUpdateRequestCommand extends HandleProtocolMessageCommand { await this.blockchainModuleManager.getFinalizationCommitsNumber(blockchain); const updateCommitDelay = await this.calculateUpdateCommitDelay( + blockchain, updateCommitWindowDuration, finalizationCommitsNumber, R0, @@ -86,7 +86,7 @@ class HandleUpdateRequestCommand extends HandleProtocolMessageCommand { }), this.commandExecutor.add({ name: 'submitUpdateCommitCommand', - delay: updateCommitDelay * 1000, + delay: updateCommitDelay, retries: COMMAND_RETRIES.SUBMIT_UPDATE_COMMIT, data: { ...commandData, @@ -108,14 +108,16 @@ class HandleUpdateRequestCommand extends HandleProtocolMessageCommand { } async calculateUpdateCommitDelay( + blockchain, updateCommitWindowDuration, finalizationCommitsNumber, R0, rank, ) { const r0OffsetPeriod = 0; + const blockTime = this.blockchainModuleManager.getBlockTimeMillis(blockchain); // wait for 5 blocks for first batch to send commits - const commitsBlockDuration = BLOCK_TIME * COMMIT_BLOCK_DURATION_IN_BLOCKS; + const commitsBlockDuration = blockTime * COMMIT_BLOCK_DURATION_IN_BLOCKS; const commitBlock = Math.floor(rank / finalizationCommitsNumber); // put 2 blocks delay between nodes if they are not in first batch const nextNodeDelay = @@ -123,10 +125,12 @@ class HandleUpdateRequestCommand extends HandleProtocolMessageCommand { ? 0 : (rank % finalizationCommitsNumber) * COMMITS_DELAY_BETWEEN_NODES_IN_BLOCKS * - BLOCK_TIME; + blockTime; const delay = commitsBlockDuration * commitBlock + r0OffsetPeriod + nextNodeDelay; this.logger.info( - `Calculated update commit delay: ${delay}, commitsBlockDuration: ${commitsBlockDuration}, commitBlock: ${commitBlock}, r0OffsetPeriod:${r0OffsetPeriod}, updateCommitWindowDuration ${updateCommitWindowDuration}, finalizationCommitsNumber: ${finalizationCommitsNumber}, r0: ${R0}, rank: ${rank}`, + `Calculated update commit delay: ${Math.floor( + delay / 1000, + )}s, commitsBlockDuration: ${commitsBlockDuration}, commitBlock: ${commitBlock}, r0OffsetPeriod:${r0OffsetPeriod}, updateCommitWindowDuration ${updateCommitWindowDuration}s, finalizationCommitsNumber: ${finalizationCommitsNumber}, r0: ${R0}, rank: ${rank}`, ); return delay; diff --git a/src/constants/constants.js b/src/constants/constants.js index ee3cdfabc..f5707fd0a 100644 --- a/src/constants/constants.js +++ b/src/constants/constants.js @@ -14,7 +14,7 @@ export const PRIVATE_ASSERTION_PREDICATE = 'https://ontology.origintrail.io/dkg/1.0#privateAssertionID'; export const COMMIT_BLOCK_DURATION_IN_BLOCKS = 5; -export const BLOCK_TIME = 12; + export const COMMITS_DELAY_BETWEEN_NODES_IN_BLOCKS = 2; export const TRANSACTION_POLLING_TIMEOUT_MILLIS = 50 * 1000; @@ -129,6 +129,7 @@ export const PERMANENT_COMMANDS = [ 'operationIdCleanerCommand', 'commandsCleanerCommand', 'dialPeersCommand', + 'epochCheckCommand', ]; export const MAX_COMMAND_DELAY_IN_MILLS = 14400 * 60 * 1000; // 10 days @@ -451,3 +452,11 @@ export const FIXED_GAS_LIMIT_METHODS = { submitUpdateCommit: 600000, sendProof: 500000, }; + +export const BLOCK_TIME_MILLIS = { + OTP: 12_000, + HARDHAT: 5_000, + DEFAULT: 12_000, +}; + +export const TRANSACTION_CONFIRMATIONS = 2; diff --git a/src/migration/service-agreements-metadata-migration.js b/src/migration/service-agreements-metadata-migration.js index 57a71d077..bcfbbeb9b 100644 --- a/src/migration/service-agreements-metadata-migration.js +++ b/src/migration/service-agreements-metadata-migration.js @@ -123,7 +123,6 @@ class ServiceAgreementsMetadataMigration extends BaseMigration { } const stateIndex = assertionIds.length - 1; const assertionId = assertionIds[stateIndex]; - // calculate keyword const keyword = this.blockchainModuleManager.encodePacked( blockchain, diff --git a/src/modules/blockchain/blockchain-module-manager.js b/src/modules/blockchain/blockchain-module-manager.js index e55df0320..b571962a8 100644 --- a/src/modules/blockchain/blockchain-module-manager.js +++ b/src/modules/blockchain/blockchain-module-manager.js @@ -338,6 +338,10 @@ class BlockchainModuleManager extends BaseModuleManager { return this.callImplementationFunction(blockchain, 'getCommitWindowDurationPerc'); } + async getEpochLength(blockchain) { + return this.callImplementationFunction(blockchain, 'getEpochLength'); + } + async getProofWindowDurationPerc(blockchain) { return this.callImplementationFunction(blockchain, 'getProofWindowDurationPerc'); } @@ -384,6 +388,10 @@ class BlockchainModuleManager extends BaseModuleManager { return this.callImplementationFunction(blockchain, 'getBlockchainTimestamp'); } + getBlockTimeMillis(blockchain) { + return this.callImplementationFunction(blockchain, 'getBlockTimeMillis'); + } + async hasPendingUpdate(blockchain, tokenId) { return this.callImplementationFunction(blockchain, 'hasPendingUpdate', [tokenId]); } diff --git a/src/modules/blockchain/implementation/hardhat/hardhat-service.js b/src/modules/blockchain/implementation/hardhat/hardhat-service.js index 5d21ba580..603a1c630 100644 --- a/src/modules/blockchain/implementation/hardhat/hardhat-service.js +++ b/src/modules/blockchain/implementation/hardhat/hardhat-service.js @@ -1,3 +1,4 @@ +import { BLOCK_TIME_MILLIS } from '../../../../constants/constants.js'; import Web3Service from '../web3-service.js'; class HardhatService extends Web3Service { @@ -12,6 +13,10 @@ class HardhatService extends Web3Service { return latestBlock.timestamp; } + getBlockTimeMillis() { + return BLOCK_TIME_MILLIS.HARDHAT; + } + async providerReady() { return this.provider.ready; } diff --git a/src/modules/blockchain/implementation/ot-parachain/ot-parachain-service.js b/src/modules/blockchain/implementation/ot-parachain/ot-parachain-service.js index 77f5854a5..87a819df6 100644 --- a/src/modules/blockchain/implementation/ot-parachain/ot-parachain-service.js +++ b/src/modules/blockchain/implementation/ot-parachain/ot-parachain-service.js @@ -1,4 +1,5 @@ import { ApiPromise, WsProvider, HttpProvider } from '@polkadot/api'; +import { BLOCK_TIME_MILLIS } from '../../../../constants/constants'; import Web3Service from '../web3-service.js'; const NATIVE_TOKEN_DECIMALS = 12; @@ -140,6 +141,10 @@ class OtParachainService extends Web3Service { const nativeBalance = await this.wallet.getBalance(); return nativeBalance / 10 ** NATIVE_TOKEN_DECIMALS; } + + getBlockTimeMillis() { + return BLOCK_TIME_MILLIS.OTP; + } } export default OtParachainService; diff --git a/src/modules/blockchain/implementation/web3-service.js b/src/modules/blockchain/implementation/web3-service.js index 5eea08430..65d900dfd 100644 --- a/src/modules/blockchain/implementation/web3-service.js +++ b/src/modules/blockchain/implementation/web3-service.js @@ -10,6 +10,8 @@ import { TRANSACTION_QUEUE_CONCURRENCY, FIXED_GAS_LIMIT_METHODS, TRANSACTION_POLLING_TIMEOUT_MILLIS, + TRANSACTION_CONFIRMATIONS, + BLOCK_TIME_MILLIS, } from '../../../constants/constants.js'; const require = createRequire(import.meta.url); @@ -243,6 +245,10 @@ class Web3Service { return latestBlock.number; } + getBlockTimeMillis() { + return BLOCK_TIME_MILLIS.DEFAULT; + } + async getIdentityId() { if (this.config.identityId) { return this.config.identityId; @@ -358,7 +364,7 @@ class Web3Service { }); result = await this.provider.waitForTransaction( tx.hash, - null, + TRANSACTION_CONFIRMATIONS, TRANSACTION_POLLING_TIMEOUT_MILLIS, ); } catch (error) { @@ -829,6 +835,15 @@ class Web3Service { ); } + async getEpochLength() { + const epochLength = await this.callContractFunction( + this.ParametersStorageContract, + 'epochLength', + [], + ); + return Number(epochLength); + } + async isHashFunction(hashFunctionId) { return this.callContractFunction(this.HashingProxyContract, 'isHashFunction(uint8)', [ hashFunctionId, diff --git a/src/modules/repository/implementation/sequelize/repositories/service-agreement-repository.js b/src/modules/repository/implementation/sequelize/repositories/service-agreement-repository.js index 8a5fde943..275bc8510 100644 --- a/src/modules/repository/implementation/sequelize/repositories/service-agreement-repository.js +++ b/src/modules/repository/implementation/sequelize/repositories/service-agreement-repository.js @@ -6,6 +6,21 @@ class ServiceAgreementRepository { this.model = models.service_agreement; } + async updateServiceAgreementEpochsNumber(agreementId, epochsNumber) { + return this.model.update( + { epochs_number: epochsNumber }, + { + where: { agreement_id: agreementId }, + }, + ); + } + + async removeServiceAgreements(agreementIds) { + return this.model.destroy({ + where: { agreement_id: { [Sequelize.Op.in]: agreementIds } }, + }); + } + async updateServiceAgreementRecord( blockchainId, contract, @@ -33,15 +48,43 @@ class ServiceAgreementRepository { epoch_length: epochLength, score_function_id: scoreFunctionId, proof_window_offset_perc: proofWindowOffsetPerc, - last_commit_epoch: lastCommitEpoch, - last_proof_epoch: lastProofEpoch, hash_function_id: hashFunctionId, keyword, assertion_id: assertionId, state_index: stateIndex, + last_commit_epoch: lastCommitEpoch, + last_proof_epoch: lastProofEpoch, }); } + async bulkCreateServiceAgreementRecords(serviceAgreements) { + return this.model.bulkCreate(serviceAgreements, { + validate: true, + }); + } + + async updateServiceAgreementLastCommitEpoch(agreementId, lastCommitEpoch) { + return this.model.update( + { last_commit_epoch: lastCommitEpoch }, + { + where: { + agreement_id: agreementId, + }, + }, + ); + } + + async updateServiceAgreementLastProofEpoch(agreementId, lastProofEpoch) { + return this.model.update( + { last_proof_epoch: lastProofEpoch }, + { + where: { + agreement_id: agreementId, + }, + }, + ); + } + async removeServiceAgreementRecord(blockchainId, contract, tokenId) { await this.model.destroy({ where: { @@ -52,18 +95,103 @@ class ServiceAgreementRepository { }); } - async updateServiceAgreementEpochsNumber(agreementId, epochsNumber) { - return this.model.update( - { epochs_number: epochsNumber }, - { - where: { agreement_id: agreementId }, + getEligibleAgreementsForSubmitCommit(timestampSeconds, blockchain, commitWindowDurationPerc) { + const currentEpoch = `FLOOR((${timestampSeconds} - start_time) / epoch_length)`; + const currentEpochPerc = `((${timestampSeconds} - start_time) % epoch_length) / epoch_length * 100`; + + return this.model.findAll({ + attributes: { + include: [ + [Sequelize.literal(currentEpoch), 'current_epoch'], + [ + Sequelize.cast( + Sequelize.literal(`${commitWindowDurationPerc} - ${currentEpochPerc}`), + 'DOUBLE', + ), + 'time_left_in_submit_commit_window', + ], + ], }, - ); + where: { + blockchain_id: blockchain, + [Sequelize.Op.or]: [ + { + last_commit_epoch: { + [Sequelize.Op.is]: null, + }, + }, + { + last_commit_epoch: { + [Sequelize.Op.lt]: Sequelize.literal(currentEpoch), + }, + }, + ], + [Sequelize.Op.and]: Sequelize.literal( + `${currentEpochPerc} < ${commitWindowDurationPerc}`, + ), + epochs_number: { + [Sequelize.Op.gt]: Sequelize.literal(currentEpoch), + }, + }, + order: [[Sequelize.col('time_left_in_submit_commit_window'), 'ASC']], + limit: 100, + raw: true, + }); } - async removeServiceAgreements(agreementIds) { - return this.model.destroy({ - where: { agreement_id: { [Sequelize.Op.in]: agreementIds } }, + async getEligibleAgreementsForSubmitProof( + timestampSeconds, + blockchain, + proofWindowDurationPerc, + ) { + const currentEpoch = `FLOOR((${timestampSeconds} - start_time) / epoch_length)`; + const currentEpochPerc = `((${timestampSeconds} - start_time) % epoch_length) / epoch_length * 100`; + + return this.model.findAll({ + attributes: { + include: [ + [Sequelize.literal(currentEpoch), 'current_epoch'], + [ + Sequelize.cast( + Sequelize.literal( + `proof_window_offset_perc + ${proofWindowDurationPerc} - ${currentEpochPerc}`, + ), + 'DOUBLE', + ), + 'time_left_in_submit_proof_window', + ], + ], + }, + where: { + blockchain_id: blockchain, + last_commit_epoch: { + [Sequelize.Op.eq]: Sequelize.literal(currentEpoch), + }, + [Sequelize.Op.or]: [ + { + last_proof_epoch: { + [Sequelize.Op.is]: null, + }, + }, + { + last_proof_epoch: { + [Sequelize.Op.lt]: Sequelize.literal(currentEpoch), + }, + }, + ], + proof_window_offset_perc: { + [Sequelize.Op.lte]: Sequelize.literal(`${currentEpochPerc}`), + [Sequelize.Op.gt]: Sequelize.literal( + `${currentEpochPerc} - ${proofWindowDurationPerc}`, + ), + }, + epochs_number: { + [Sequelize.Op.gt]: Sequelize.literal(currentEpoch), + }, + }, + order: [[Sequelize.col('time_left_in_submit_proof_window'), 'ASC']], + limit: 100, + raw: true, }); } } diff --git a/src/modules/repository/implementation/sequelize/sequelize-repository.js b/src/modules/repository/implementation/sequelize/sequelize-repository.js index 2953c6770..f232c34f8 100644 --- a/src/modules/repository/implementation/sequelize/sequelize-repository.js +++ b/src/modules/repository/implementation/sequelize/sequelize-repository.js @@ -128,6 +128,10 @@ class SequelizeRepository { async query(query) { return this.models.sequelize.query(query); } + + async destroyAllRecords(table) { + return this.models[table].destroy({ where: {} }); + } } export default SequelizeRepository; diff --git a/src/modules/repository/repository-module-manager.js b/src/modules/repository/repository-module-manager.js index 42f0a2bbb..c331516d8 100644 --- a/src/modules/repository/repository-module-manager.js +++ b/src/modules/repository/repository-module-manager.js @@ -30,6 +30,12 @@ class RepositoryModuleManager extends BaseModuleManager { } } + async destroyAllRecords(table) { + if (this.initialized) { + return this.getImplementation().module.destroyAllRecords(table); + } + } + async updateCommand(update, opts) { return this.getRepository('command').updateCommand(update, opts); } @@ -266,31 +272,89 @@ class RepositoryModuleManager extends BaseModuleManager { lastCommitEpoch, lastProofEpoch, ) { - return this.getRepository('service_agreement').updateServiceAgreementRecord( - blockchainId, - contract, - tokenId, - agreementId, - startTime, - epochsNumber, - epochLength, - scoreFunctionId, - proofWindowOffsetPerc, - hashFunctionId, - keyword, - assertionId, - stateIndex, - lastCommitEpoch, - lastProofEpoch, - ); + if (this.initialized) { + return this.getRepository('service_agreement').updateServiceAgreementRecord( + blockchainId, + contract, + tokenId, + agreementId, + startTime, + epochsNumber, + epochLength, + scoreFunctionId, + proofWindowOffsetPerc, + hashFunctionId, + keyword, + assertionId, + stateIndex, + lastCommitEpoch, + lastProofEpoch, + ); + } + } + + async bulkCreateServiceAgreementRecords(records) { + if (this.initialized) { + return this.getRepository('service_agreement').bulkCreateServiceAgreementRecords( + records, + ); + } + } + + async updateServiceAgreementLastCommitEpoch(agreementId, lastCommitEpoch) { + if (this.initialized) { + return this.getRepository('service_agreement').updateServiceAgreementLastCommitEpoch( + agreementId, + lastCommitEpoch, + ); + } + } + + async updateServiceAgreementLastProofEpoch(agreementId, lastProofEpoch) { + if (this.initialized) { + return this.getRepository('service_agreement').updateServiceAgreementLastProofEpoch( + agreementId, + lastProofEpoch, + ); + } } async removeServiceAgreementRecord(blockchainId, contract, tokenId) { - return this.getRepository('service_agreement').removeServiceAgreementRecord( - blockchainId, - contract, - tokenId, - ); + if (this.initialized) { + return this.getRepository('service_agreement').removeServiceAgreementRecord( + blockchainId, + contract, + tokenId, + ); + } + } + + async getEligibleAgreementsForSubmitCommit( + timestampSeconds, + blockchain, + commitWindowDurationPerc, + ) { + if (this.initialized) { + return this.getRepository('service_agreement').getEligibleAgreementsForSubmitCommit( + timestampSeconds, + blockchain, + commitWindowDurationPerc, + ); + } + } + + async getEligibleAgreementsForSubmitProof( + timestampSeconds, + blockchain, + proofWindowDurationPerc, + ) { + if (this.initialized) { + return this.getRepository('service_agreement').getEligibleAgreementsForSubmitProof( + timestampSeconds, + blockchain, + proofWindowDurationPerc, + ); + } } async removeServiceAgreements(agreementIds) { diff --git a/test/modules/repository/config.json b/test/modules/repository/config.json new file mode 100644 index 000000000..0a54659e2 --- /dev/null +++ b/test/modules/repository/config.json @@ -0,0 +1,22 @@ +{ + "modules": { + "repository": { + "enabled": true, + "implementation": { + "sequelize-repository": { + "enabled": true, + "package": "./repository/implementation/sequelize/sequelize-repository.js", + "config": { + "database": "operationaldb-test", + "user": "root", + "password": "", + "port": "3306", + "host": "localhost", + "dialect": "mysql", + "logging": false + } + } + } + } + } +} diff --git a/test/modules/repository/repository.js b/test/modules/repository/repository.js new file mode 100644 index 000000000..3f9a78d7d --- /dev/null +++ b/test/modules/repository/repository.js @@ -0,0 +1,460 @@ +import { utils } from 'ethers'; +import { describe, it, before, beforeEach, afterEach, after } from 'mocha'; +import { expect, assert } from 'chai'; +import { readFile } from 'fs/promises'; +import Logger from '../../../src/logger/logger.js'; +import RepositoryModuleManager from '../../../src/modules/repository/repository-module-manager.js'; + +let logger; +let repositoryModuleManager; +const config = JSON.parse(await readFile('./test/modules/repository/config.json')); + +const blockchain = 'hardhat'; +const createAgreement = ({ + blockchain_id = blockchain, + asset_storage_contract_address = '0xB0D4afd8879eD9F52b28595d31B441D079B2Ca07', + token_id, + agreement_id = null, + start_time, + epochs_number = 2, + epoch_length = 100, + score_function_id = 1, + proof_window_offset_perc = 66, + hash_function_id = 1, + keyword = '0xB0D4afd8879eD9F52b28595d31B441D079B2Ca0768e44dc71bf509adfccbea9df949f253afa56796a3a926203f90a1e4914247d3', + assertion_id = '0x68e44dc71bf509adfccbea9df949f253afa56796a3a926203f90a1e4914247d3', + state_index = 1, + last_commit_epoch = null, + last_proof_epoch = null, +}) => { + const agreementId = + agreement_id ?? + utils.sha256( + utils.toUtf8Bytes( + utils.solidityPack( + ['address', 'uint256', 'bytes'], + [asset_storage_contract_address, token_id, keyword], + ), + ), + ); + + return { + blockchain_id, + asset_storage_contract_address, + token_id, + agreement_id: agreementId, + start_time, + epochs_number, + epoch_length, + score_function_id, + proof_window_offset_perc, + hash_function_id, + keyword, + assertion_id, + state_index, + last_commit_epoch, + last_proof_epoch, + }; +}; + +describe('Repository module', () => { + before('Initialize repository module manager', async function initializeRepository() { + this.timeout(30_000); + logger = new Logger('trace'); + logger.info = () => {}; + repositoryModuleManager = new RepositoryModuleManager({ config, logger }); + await repositoryModuleManager.initialize(); + await repositoryModuleManager.destroyAllRecords('service_agreement'); + }); + + afterEach('Destroy all records', async function destroyAllRecords() { + this.timeout(30_000); + await repositoryModuleManager.destroyAllRecords('service_agreement'); + }); + + after(async function dropDatabase() { + this.timeout(30_000); + await repositoryModuleManager.dropDatabase(); + }); + + describe('Empty repository', () => { + it('returns empty list if no service agreements', async () => { + const eligibleAgreements = + await repositoryModuleManager.getEligibleAgreementsForSubmitCommit( + Date.now(), + blockchain, + 25, + ); + + assert(expect(eligibleAgreements).to.exist); + expect(eligibleAgreements).to.be.instanceOf(Array); + expect(eligibleAgreements).to.have.length(0); + }); + }); + describe('Insert and update service agreement', () => { + const agreement = { + blockchain_id: blockchain, + asset_storage_contract_address: '0xB0D4afd8879eD9F52b28595d31B441D079B2Ca07', + token_id: 0, + agreement_id: '0x44cf660357e2d7462c25fd8e50b68abe332d7a70b07a76e92f628846ea585881', + start_time: 1683032289, + epochs_number: 2, + epoch_length: 360, + score_function_id: 1, + proof_window_offset_perc: 66, + hash_function_id: 1, + keyword: + '0xB0D4afd8879eD9F52b28595d31B441D079B2Ca0768e44dc71bf509adfccbea9df949f253afa56796a3a926203f90a1e4914247d3', + assertion_id: '0x68e44dc71bf509adfccbea9df949f253afa56796a3a926203f90a1e4914247d3', + state_index: 1, + }; + + it('inserts service agreement', async () => { + const inserted = await repositoryModuleManager.updateServiceAgreementRecord( + agreement.blockchain_id, + agreement.asset_storage_contract_address, + agreement.token_id, + agreement.agreement_id, + agreement.start_time, + agreement.epochs_number, + agreement.epoch_length, + agreement.score_function_id, + agreement.proof_window_offset_perc, + agreement.hash_function_id, + agreement.keyword, + agreement.assertion_id, + agreement.state_index, + agreement.last_commit_epoch, + agreement.last_proof_epoch, + ); + const row = inserted[0]?.dataValues; + + assert(expect(row).to.exist); + expect(row.blockchain_id).to.equal(agreement.blockchain_id); + expect(row.asset_storage_contract_address).to.equal( + agreement.asset_storage_contract_address, + ); + expect(row.token_id).to.equal(agreement.token_id); + expect(row.agreement_id).to.equal(agreement.agreement_id); + expect(row.start_time).to.equal(agreement.start_time); + expect(row.epochs_number).to.equal(agreement.epochs_number); + expect(row.epoch_length).to.equal(agreement.epoch_length); + expect(row.score_function_id).to.equal(agreement.score_function_id); + expect(row.proof_window_offset_perc).to.equal(agreement.proof_window_offset_perc); + expect(row.hash_function_id).to.equal(agreement.hash_function_id); + expect(row.keyword).to.equal(agreement.keyword); + expect(row.assertion_id).to.equal(agreement.assertion_id); + expect(row.state_index).to.equal(agreement.state_index); + assert(expect(row.last_commit_epoch).to.not.exist); + assert(expect(row.last_proof_epoch).to.not.exist); + }); + }); + + describe('Eligible service agreements', () => { + const agreements = [ + createAgreement({ token_id: 0, start_time: 0 }), + createAgreement({ + token_id: 1, + start_time: 15, + last_commit_epoch: 0, + }), + createAgreement({ token_id: 2, start_time: 25 }), + createAgreement({ + token_id: 3, + start_time: 25, + last_commit_epoch: 0, + last_proof_epoch: 0, + }), + createAgreement({ token_id: 4, start_time: 49 }), + ]; + + beforeEach(async () => { + await Promise.all( + agreements.map((agreement) => + repositoryModuleManager.updateServiceAgreementRecord( + agreement.blockchain_id, + agreement.asset_storage_contract_address, + agreement.token_id, + agreement.agreement_id, + agreement.start_time, + agreement.epochs_number, + agreement.epoch_length, + agreement.score_function_id, + agreement.proof_window_offset_perc, + agreement.hash_function_id, + agreement.keyword, + agreement.assertion_id, + agreement.state_index, + agreement.last_commit_epoch, + agreement.last_proof_epoch, + ), + ), + ); + }); + + describe('getEligibleAgreementsForSubmitCommit returns correct agreements', () => { + const testEligibleAgreementsForSubmitCommit = + (currentTimestamp, commitWindowDurationPerc, expectedAgreements) => async () => { + const eligibleAgreements = + await repositoryModuleManager.getEligibleAgreementsForSubmitCommit( + currentTimestamp, + blockchain, + commitWindowDurationPerc, + ); + + assert(expect(eligibleAgreements).to.exist); + expect(eligibleAgreements).to.be.instanceOf(Array); + expect(eligibleAgreements).to.have.length(expectedAgreements.length); + expect(eligibleAgreements).to.have.deep.members(expectedAgreements); + + // ensure order is correct + for (let i = 0; i < eligibleAgreements.length; i += 1) { + assert.strictEqual( + eligibleAgreements[i].time_left_in_submit_commit_window, + expectedAgreements[i].time_left_in_submit_commit_window, + ); + } + }; + + it( + 'returns two eligible service agreements at timestamp 49', + testEligibleAgreementsForSubmitCommit(49, 25, [ + { ...agreements[2], current_epoch: 0, time_left_in_submit_commit_window: 1 }, + { ...agreements[4], current_epoch: 0, time_left_in_submit_commit_window: 25 }, + ]), + ); + it( + 'returns one eligible service agreement at timestamp 51', + testEligibleAgreementsForSubmitCommit(51, 25, [ + { ...agreements[4], current_epoch: 0, time_left_in_submit_commit_window: 23 }, + ]), + ); + it( + 'returns no eligible service agreement at timestamp 74', + testEligibleAgreementsForSubmitCommit(74, 25, []), + ); + it( + 'returns no eligible service agreements at timestamp 75', + testEligibleAgreementsForSubmitCommit(75, 25, []), + ); + it( + 'returns one eligible service agreements at timestamp 100', + testEligibleAgreementsForSubmitCommit(100, 25, [ + { ...agreements[0], current_epoch: 1, time_left_in_submit_commit_window: 25 }, + ]), + ); + it( + 'returns two eligible service agreements at timestamp 124', + testEligibleAgreementsForSubmitCommit(124, 25, [ + { ...agreements[0], current_epoch: 1, time_left_in_submit_commit_window: 1 }, + { ...agreements[1], current_epoch: 1, time_left_in_submit_commit_window: 16 }, + ]), + ); + it( + 'returns three eligible service agreements at timestamp 125', + testEligibleAgreementsForSubmitCommit(125, 25, [ + { ...agreements[1], current_epoch: 1, time_left_in_submit_commit_window: 15 }, + { ...agreements[2], current_epoch: 1, time_left_in_submit_commit_window: 25 }, + { ...agreements[3], current_epoch: 1, time_left_in_submit_commit_window: 25 }, + ]), + ); + it( + 'returns three eligible service agreements at timestamp 126', + testEligibleAgreementsForSubmitCommit(126, 25, [ + { ...agreements[1], current_epoch: 1, time_left_in_submit_commit_window: 14 }, + { ...agreements[2], current_epoch: 1, time_left_in_submit_commit_window: 24 }, + { ...agreements[3], current_epoch: 1, time_left_in_submit_commit_window: 24 }, + ]), + ); + it( + 'returns three eligible service agreements at timestamp 149', + testEligibleAgreementsForSubmitCommit(149, 25, [ + { ...agreements[2], current_epoch: 1, time_left_in_submit_commit_window: 1 }, + { ...agreements[3], current_epoch: 1, time_left_in_submit_commit_window: 1 }, + { ...agreements[4], current_epoch: 1, time_left_in_submit_commit_window: 25 }, + ]), + ); + it( + 'returns one eligible service agreements at timestamp 151', + testEligibleAgreementsForSubmitCommit(151, 25, [ + { ...agreements[4], current_epoch: 1, time_left_in_submit_commit_window: 23 }, + ]), + ); + it( + 'returns no eligible service agreements at timestamp 175', + testEligibleAgreementsForSubmitCommit(175, 25, []), + ); + it( + 'returns no eligible service agreements at timestamp 225', + testEligibleAgreementsForSubmitCommit(225, 25, []), + ); + }); + + describe('getEligibleAgreementsForSubmitProof returns correct agreements', () => { + const testEligibleAgreementsForSubmitProof = + (currentTimestamp, proofWindowDurationPerc, expectedAgreements) => async () => { + const eligibleAgreements = + await repositoryModuleManager.getEligibleAgreementsForSubmitProof( + currentTimestamp, + blockchain, + proofWindowDurationPerc, + ); + + assert(expect(eligibleAgreements).to.exist); + expect(eligibleAgreements).to.be.instanceOf(Array); + expect(eligibleAgreements).to.have.length(expectedAgreements.length); + expect(eligibleAgreements).to.have.deep.members(expectedAgreements); + + // ensure order is correct + for (let i = 0; i < eligibleAgreements.length; i += 1) { + assert.strictEqual( + eligibleAgreements[i].time_left_in_submit_proof_window, + expectedAgreements[i].time_left_in_submit_proof_window, + ); + } + }; + + it( + 'returns no eligible service agreement at timestamp 49', + testEligibleAgreementsForSubmitProof(49, 33, []), + ); + it( + 'returns no eligible service agreement at timestamp 67', + testEligibleAgreementsForSubmitProof(67, 33, []), + ); + it( + 'returns no eligible service agreement at timestamp 80', + testEligibleAgreementsForSubmitProof(80, 33, []), + ); + it( + 'returns one eligible service agreements at timestamp 81', + testEligibleAgreementsForSubmitProof(81, 33, [ + { ...agreements[1], current_epoch: 0, time_left_in_submit_proof_window: 33 }, + ]), + ); + it( + 'returns one eligible service agreements at timestamp 92', + testEligibleAgreementsForSubmitProof(92, 33, [ + { ...agreements[1], current_epoch: 0, time_left_in_submit_proof_window: 22 }, + ]), + ); + it( + 'returns one eligible service agreements at timestamp 113', + testEligibleAgreementsForSubmitProof(113, 33, [ + { ...agreements[1], current_epoch: 0, time_left_in_submit_proof_window: 1 }, + ]), + ); + it( + 'returns no eligible service agreements at timestamp 114', + testEligibleAgreementsForSubmitProof(114, 33, []), + ); + it( + 'returns no eligible service agreements at timestamp 167', + testEligibleAgreementsForSubmitProof(167, 33, []), + ); + it( + 'returns no eligible service agreements at timestamp 181', + testEligibleAgreementsForSubmitProof(181, 33, []), + ); + it( + 'returns no eligible service agreements at timestamp 192', + testEligibleAgreementsForSubmitProof(192, 33, []), + ); + it( + 'returns no eligible service agreements at timestamp 199', + testEligibleAgreementsForSubmitProof(199, 33, []), + ); + it( + 'returns no eligible service agreements at timestamp 200', + testEligibleAgreementsForSubmitProof(200, 33, []), + ); + }); + }); + + async function insertLoadTestAgreements(numAgreements) { + let agreements = []; + for (let tokenId = 0; tokenId < numAgreements; tokenId += 1) { + agreements.push( + createAgreement({ + token_id: tokenId, + start_time: Math.floor(Math.random() * 101), + last_commit_epoch: [null, 0][Math.floor(Math.random() * 3)], + last_proof_epoch: [null, 0][Math.floor(Math.random() * 3)], + }), + ); + + if (agreements.length % 100_000 === 0) { + // eslint-disable-next-line no-await-in-loop + await repositoryModuleManager.bulkCreateServiceAgreementRecords(agreements); + agreements = []; + } + } + if (agreements.length) { + await repositoryModuleManager.bulkCreateServiceAgreementRecords(agreements); + } + } + + describe('test load', () => { + describe('100_000 rows', () => { + beforeEach(async function t() { + this.timeout(0); + await insertLoadTestAgreements(100_000); + }); + + it('getEligibleAgreementsForSubmitCommit returns agreements in less than 100 ms', async () => { + const start = performance.now(); + await repositoryModuleManager.getEligibleAgreementsForSubmitCommit( + 100, + blockchain, + 25, + ); + const end = performance.now(); + const duration = end - start; + expect(duration).to.be.lessThan(100); + }); + + it('getEligibleAgreementsForSubmitProof returns agreements in less than 100 ms', async () => { + const start = performance.now(); + await repositoryModuleManager.getEligibleAgreementsForSubmitProof( + 100, + blockchain, + 33, + ); + const end = performance.now(); + const duration = end - start; + expect(duration).to.be.lessThan(100); + }); + }); + + describe('1_000_000 rows', () => { + beforeEach(async function t() { + this.timeout(0); + await insertLoadTestAgreements(1_000_000); + }); + + it('getEligibleAgreementsForSubmitCommit returns agreements in less than 1000 ms', async () => { + const start = performance.now(); + await repositoryModuleManager.getEligibleAgreementsForSubmitCommit( + 100, + blockchain, + 25, + ); + const end = performance.now(); + const duration = end - start; + expect(duration).to.be.lessThan(1000); + }); + + it('getEligibleAgreementsForSubmitProof returns agreements in less than 1000 ms', async () => { + const start = performance.now(); + await repositoryModuleManager.getEligibleAgreementsForSubmitProof( + 100, + blockchain, + 33, + ); + const end = performance.now(); + const duration = end - start; + expect(duration).to.be.lessThan(1000); + }); + }); + }); +}); diff --git a/test/modules/triple-store/triple-store.js b/test/modules/triple-store/triple-store.js index 2f4cbf6e3..9209cf8bc 100644 --- a/test/modules/triple-store/triple-store.js +++ b/test/modules/triple-store/triple-store.js @@ -1,4 +1,4 @@ -import { describe, it, before, beforeEach } from 'mocha'; +/* import { describe, it, before, beforeEach } from 'mocha'; import chai from 'chai'; import { readFile } from 'fs/promises'; import { formatAssertion, calculateRoot } from 'assertion-tools'; @@ -72,3 +72,4 @@ describe('Triple store module', () => { } }); }); + */ From 9683e33e351b602ace3173a9941e99577499ff84 Mon Sep 17 00:00:00 2001 From: zeroxbt <89495162+zeroxbt@users.noreply.github.com> Date: Tue, 16 May 2023 13:34:52 +0200 Subject: [PATCH 04/16] add remove old epoch commands migration (#2506) --- ot-node.js | 24 +++++++++++++++++- .../remove-old-epoch-commands-migration.js | 25 +++++++++++++++++++ 2 files changed, 48 insertions(+), 1 deletion(-) create mode 100644 src/migration/remove-old-epoch-commands-migration.js diff --git a/ot-node.js b/ot-node.js index a11a1c998..17407c389 100644 --- a/ot-node.js +++ b/ot-node.js @@ -16,6 +16,7 @@ import ServiceAgreementsMetadataMigration from './src/migration/service-agreemen import RemoveAgreementStartEndTimeMigration from './src/migration/remove-agreement-start-end-time-migration.js'; import MarkOldBlockchainEventsAsProcessedMigration from './src/migration/mark-old-blockchain-events-as-processed-migration.js'; import TripleStoreMetadataMigration from './src/migration/triple-store-metadata-migration.js'; +import RemoveOldEpochCommandsMigration from './src/migration/remove-old-epoch-commands-migration.js'; const require = createRequire(import.meta.url); const pjson = require('./package.json'); @@ -55,7 +56,8 @@ class OTNode { await this.executeRemoveAgreementStartEndTimeMigration(); await this.executeMarkOldBlockchainEventsAsProcessedMigration(); await this.executeTripleStoreMetadataMigration(); - this.executeServiceAgreementsMetadataMigration(); + await this.executeServiceAgreementsMetadataMigration(); + await this.executeRemoveOldEpochCommandsMigration(); await this.createProfiles(); @@ -399,6 +401,26 @@ class OTNode { } } + async executeRemoveOldEpochCommandsMigration() { + if ( + process.env.NODE_ENV === NODE_ENVIRONMENTS.DEVELOPMENT || + process.env.NODE_ENV === NODE_ENVIRONMENTS.TEST + ) + return; + + const repositoryModuleManager = this.container.resolve('repositoryModuleManager'); + + const migration = new RemoveOldEpochCommandsMigration( + 'removeOldEpochCommandsMigration', + this.logger, + this.config, + repositoryModuleManager, + ); + if (!(await migration.migrationAlreadyExecuted())) { + await migration.migrate(); + } + } + async initializeShardingTableService() { try { const shardingTableService = this.container.resolve('shardingTableService'); diff --git a/src/migration/remove-old-epoch-commands-migration.js b/src/migration/remove-old-epoch-commands-migration.js new file mode 100644 index 000000000..bda9b823b --- /dev/null +++ b/src/migration/remove-old-epoch-commands-migration.js @@ -0,0 +1,25 @@ +import BaseMigration from './base-migration.js'; + +class RemoveOldEpochCommandsMigration extends BaseMigration { + constructor(migrationName, logger, config, repositoryModuleManager) { + super(migrationName, logger, config); + this.repositoryModuleManager = repositoryModuleManager; + } + + async executeMigration() { + const commandsToDestroy = [ + 'epochCheckCommand', + 'calculateProofsCommand', + 'submitCommitCommand', + 'submitProofsCommand', + ]; + + await Promise.all( + commandsToDestroy.map((command) => + this.repositoryModuleManager.destroyCommand(command), + ), + ); + } +} + +export default RemoveOldEpochCommandsMigration; From 4429ea4aa2991328d7643dda8a952942ce88e5ee Mon Sep 17 00:00:00 2001 From: zeroxbt <89495162+zeroxbt@users.noreply.github.com> Date: Tue, 16 May 2023 13:45:51 +0200 Subject: [PATCH 05/16] bump version to 6.0.9 (#2523) --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 7cb3b52d7..7d1bc7066 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "origintrail_node", - "version": "6.0.8", + "version": "6.0.9", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "origintrail_node", - "version": "6.0.8", + "version": "6.0.9", "license": "ISC", "dependencies": { "@comunica/query-sparql": "^2.4.3", diff --git a/package.json b/package.json index 8f2ef8015..9a9ee6a17 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "origintrail_node", - "version": "6.0.8", + "version": "6.0.9", "description": "OTNode V6", "main": "index.js", "type": "module", From b1327852d943b43b7e6297d83c2067db9a706424 Mon Sep 17 00:00:00 2001 From: zeroxbt <89495162+zeroxbt@users.noreply.github.com> Date: Tue, 16 May 2023 14:44:10 +0200 Subject: [PATCH 06/16] fix import statement for constants (#2524) --- .../implementation/ot-parachain/ot-parachain-service.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/modules/blockchain/implementation/ot-parachain/ot-parachain-service.js b/src/modules/blockchain/implementation/ot-parachain/ot-parachain-service.js index 87a819df6..fda59af7c 100644 --- a/src/modules/blockchain/implementation/ot-parachain/ot-parachain-service.js +++ b/src/modules/blockchain/implementation/ot-parachain/ot-parachain-service.js @@ -1,5 +1,5 @@ import { ApiPromise, WsProvider, HttpProvider } from '@polkadot/api'; -import { BLOCK_TIME_MILLIS } from '../../../../constants/constants'; +import { BLOCK_TIME_MILLIS } from '../../../../constants/constants.js'; import Web3Service from '../web3-service.js'; const NATIVE_TOKEN_DECIMALS = 12; From 40fe87d170dc5af980266de0dff5eb496086c6bf Mon Sep 17 00:00:00 2001 From: zeroxbt <89495162+zeroxbt@users.noreply.github.com> Date: Tue, 16 May 2023 16:03:06 +0200 Subject: [PATCH 07/16] lower transaction confirmations to 1 (#2527) --- src/constants/constants.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/constants/constants.js b/src/constants/constants.js index f5707fd0a..2abc0abae 100644 --- a/src/constants/constants.js +++ b/src/constants/constants.js @@ -459,4 +459,4 @@ export const BLOCK_TIME_MILLIS = { DEFAULT: 12_000, }; -export const TRANSACTION_CONFIRMATIONS = 2; +export const TRANSACTION_CONFIRMATIONS = 1; From 3b98249223ee5d51dec4b90c843c595e48e6ee69 Mon Sep 17 00:00:00 2001 From: zeroxbt <89495162+zeroxbt@users.noreply.github.com> Date: Wed, 17 May 2023 12:57:23 +0200 Subject: [PATCH 08/16] fix rank calculation when node not in r2 (#2529) * fix rank calculation when node not in r2 * fix update last commit epoch * add check for commit already submitted * handle cases where node is not in neighborhood * add logs for transaction queue length * update scheduled tx logs * bump version --- package-lock.json | 4 +- package.json | 2 +- .../protocols/common/epoch-check-command.js | 59 ++++++------ .../protocols/common/submit-commit-command.js | 48 ++++++---- .../protocols/common/submit-proofs-command.js | 20 ++++- .../receiver/submit-update-commit-command.js | 29 +++--- .../v1-0-0-handle-update-request-command.js | 89 ++++++++++--------- .../blockchain/blockchain-module-manager.js | 4 + .../blockchain/implementation/web3-service.js | 6 +- 9 files changed, 157 insertions(+), 104 deletions(-) diff --git a/package-lock.json b/package-lock.json index 7d1bc7066..433b657a4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "origintrail_node", - "version": "6.0.9", + "version": "6.0.9+hotfix.1", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "origintrail_node", - "version": "6.0.9", + "version": "6.0.9+hotfix.1", "license": "ISC", "dependencies": { "@comunica/query-sparql": "^2.4.3", diff --git a/package.json b/package.json index 9a9ee6a17..86e12dbd6 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "origintrail_node", - "version": "6.0.9", + "version": "6.0.9+hotfix.1", "description": "OTNode V6", "main": "index.js", "type": "module", diff --git a/src/commands/protocols/common/epoch-check-command.js b/src/commands/protocols/common/epoch-check-command.js index 214c0cdce..91a3381b1 100644 --- a/src/commands/protocols/common/epoch-check-command.js +++ b/src/commands/protocols/common/epoch-check-command.js @@ -52,6 +52,9 @@ class EpochCheckCommand extends Command { commitWindowDurationPerc, ); + const r0 = await this.blockchainModuleManager.getR0(blockchain); + const r2 = await this.blockchainModuleManager.getR2(blockchain); + const scheduleSubmitCommitCommands = []; const updateServiceAgreementsLastCommitEpoch = []; for (const serviceAgreement of eligibleAgreementForSubmitCommit) { @@ -61,37 +64,39 @@ class EpochCheckCommand extends Command { blockchain, serviceAgreement.keyword, serviceAgreement.hash_function_id, + r2, ); - const r0 = await this.blockchainModuleManager.getR0(blockchain); + updateServiceAgreementsLastCommitEpoch.push( + this.repositoryModuleManager.updateServiceAgreementLastCommitEpoch( + serviceAgreement.agreement_id, + serviceAgreement.current_epoch, + ), + ); - if (rank < r0) { + if (rank == null) { this.logger.trace( - `Calculated rank: ${ - rank + 1 - } lower than R0: ${r0}. Scheduling submit commit command for agreement id: ${ - serviceAgreement.agreement_id - }`, - ); - scheduleSubmitCommitCommands.push( - this.scheduleSubmitCommitCommand(serviceAgreement), + `Node not in R2: ${r2} for agreement id: ${serviceAgreement.agreement_id}. Skipping scheduling submit commit command-`, ); - } else { + continue; + } + + if (rank >= r0) { this.logger.trace( - `Calculated rank: ${ - rank + 1 - } higher than R0: ${r0}. Skipping scheduling submit commit command for agreement id: ${ + `Calculated rank: ${rank + 1}. Node not in R0: ${r0} for agreement id: ${ serviceAgreement.agreement_id - }`, + }. Skipping scheduling submit commit command.`, ); + continue; } - updateServiceAgreementsLastCommitEpoch.push( - this.repositoryModuleManager.updateServiceAgreementLastCommitEpoch( - serviceAgreement.agreement_id, - serviceAgreement.current_epoch, - ), + this.logger.trace( + `Calculated rank: ${rank + 1}. Node in R0: ${r0} for agreement id: ${ + serviceAgreement.agreement_id + }. Scheduling submit commit command.`, ); + + scheduleSubmitCommitCommands.push(this.scheduleSubmitCommitCommand(serviceAgreement)); } await Promise.all([ ...scheduleSubmitCommitCommands, @@ -144,16 +149,20 @@ class EpochCheckCommand extends Command { ]); } - async calculateRank(blockchain, keyword, hashFunctionId) { - const r2 = await this.blockchainModuleManager.getR2(blockchain); + async calculateRank(blockchain, keyword, hashFunctionId, r2) { const neighbourhood = await this.shardingTableService.findNeighbourhood( blockchain, keyword, r2, hashFunctionId, - false, + true, ); + const peerId = this.networkModuleManager.getPeerId().toB58String(); + if (!neighbourhood.some((node) => node.peer_id === peerId)) { + return; + } + const scores = await Promise.all( neighbourhood.map(async (node) => ({ score: await this.serviceAgreementService.calculateScore( @@ -168,9 +177,7 @@ class EpochCheckCommand extends Command { scores.sort((a, b) => b.score - a.score); - return scores.findIndex( - (node) => node.peerId === this.networkModuleManager.getPeerId().toB58String(), - ); + return scores.findIndex((node) => node.peerId === peerId); } async isEligibleForRewards(blockchain, agreementId, epoch, stateIndex) { diff --git a/src/commands/protocols/common/submit-commit-command.js b/src/commands/protocols/common/submit-commit-command.js index fc74f34b4..f32f5ad5d 100644 --- a/src/commands/protocols/common/submit-commit-command.js +++ b/src/commands/protocols/common/submit-commit-command.js @@ -27,6 +27,14 @@ class SubmitCommitCommand extends Command { // TODO: review operationId const operationId = uuidv4(); + this.logger.trace( + `Started ${command.name} for agreement id: ${agreementId} ` + + `blockchain: ${blockchain}, contract: ${contract}, token id: ${tokenId},` + + `keyword: ${keyword}, hash function id: ${hashFunctionId}, epoch: ${epoch}, ` + + `stateIndex: ${stateIndex}, operationId: ${operationId}, ` + + ` Retry number ${COMMAND_RETRIES.SUBMIT_COMMIT - command.retries + 1}`, + ); + if (command.retries === COMMAND_RETRIES.SUBMIT_COMMIT) { this.operationIdService.emitChangeEvent( OPERATION_ID_STATUS.COMMIT_PROOF.SUBMIT_COMMIT_START, @@ -36,27 +44,18 @@ class SubmitCommitCommand extends Command { ); } - this.logger.trace( - `Started ${command.name} for agreement id: ${agreementId} ` + - `contract: ${contract}, token id: ${tokenId}, keyword: ${keyword}, ` + - `hash function id: ${hashFunctionId}, epoch: ${epoch}, stateIndex: ${stateIndex}. Retry number ${ - COMMAND_RETRIES.SUBMIT_COMMIT - command.retries + 1 - }`, + // this can happen in case node has already submitted update commit + const alreadySubmitted = await this.commitAlreadySubmitted( + blockchain, + agreementId, + epoch, + stateIndex, ); - - if (command.retries !== COMMAND_RETRIES.SUBMIT_COMMIT) { - const alreadySubmitted = await this.commitAlreadySubmitted( - blockchain, - agreementId, - epoch, - stateIndex, + if (alreadySubmitted) { + this.logger.trace( + `Commit already submitted for blockchain: ${blockchain} agreement id: ${agreementId}, epoch: ${epoch}, state index: ${stateIndex}`, ); - if (alreadySubmitted) { - this.logger.trace( - `Commit already submitted for blockchain: ${blockchain} agreement id: ${agreementId}, epoch: ${epoch}, state index: ${stateIndex}`, - ); - return Command.empty(); - } + return Command.empty(); } await this.blockchainModuleManager.submitCommit( @@ -109,6 +108,17 @@ class SubmitCommitCommand extends Command { }, ); + const transactionQueueLength = + this.blockchainModuleManager.getTransactionQueueLength(blockchain); + + this.logger.trace( + `Scheduled submit commit transaction for agreement id: ${agreementId} ` + + `blockchain: ${blockchain}, contract: ${contract}, token id: ${tokenId},` + + `keyword: ${keyword}, hash function id: ${hashFunctionId}, epoch: ${epoch}, ` + + `stateIndex: ${stateIndex}, operationId: ${operationId}, ` + + `transaction queue length: ${transactionQueueLength}.`, + ); + return Command.empty(); } diff --git a/src/commands/protocols/common/submit-proofs-command.js b/src/commands/protocols/common/submit-proofs-command.js index 8ac922ff2..16fef93d1 100644 --- a/src/commands/protocols/common/submit-proofs-command.js +++ b/src/commands/protocols/common/submit-proofs-command.js @@ -39,10 +39,10 @@ class SubmitProofsCommand extends Command { this.logger.trace( `Started ${command.name} for agreement id: ${agreementId} ` + - `contract: ${contract}, token id: ${tokenId}, keyword: ${keyword}, ` + - `hash function id: ${hashFunctionId}, stateIndex: ${stateIndex}. Retry number ${ - COMMAND_RETRIES.SUBMIT_PROOFS - command.retries + 1 - }`, + `blockchain: ${blockchain}, contract: ${contract}, token id: ${tokenId},` + + `keyword: ${keyword}, hash function id: ${hashFunctionId}, epoch: ${epoch}, ` + + `stateIndex: ${stateIndex}, operationId: ${operationId}, ` + + ` Retry number ${COMMAND_RETRIES.SUBMIT_PROOFS - command.retries + 1}`, ); if (command.retries === COMMAND_RETRIES.SUBMIT_PROOFS) { @@ -159,6 +159,18 @@ class SubmitProofsCommand extends Command { } }, ); + + const transactionQueueLength = + this.blockchainModuleManager.getTransactionQueueLength(blockchain); + + this.logger.trace( + `Scheduled send proof transaction for agreement id: ${agreementId} ` + + `blockchain: ${blockchain}, contract: ${contract}, token id: ${tokenId},` + + `keyword: ${keyword}, hash function id: ${hashFunctionId}, epoch: ${epoch}, ` + + `stateIndex: ${stateIndex}, operationId: ${operationId}, ` + + `transaction queue length: ${transactionQueueLength}.`, + ); + return Command.empty(); } diff --git a/src/commands/protocols/update/receiver/submit-update-commit-command.js b/src/commands/protocols/update/receiver/submit-update-commit-command.js index 2b36c1d30..682e16aa2 100644 --- a/src/commands/protocols/update/receiver/submit-update-commit-command.js +++ b/src/commands/protocols/update/receiver/submit-update-commit-command.js @@ -10,10 +10,7 @@ class SubmitUpdateCommitCommand extends Command { super(ctx); this.commandExecutor = ctx.commandExecutor; this.blockchainModuleManager = ctx.blockchainModuleManager; - this.serviceAgreementService = ctx.serviceAgreementService; this.operationIdService = ctx.operationIdService; - this.shardingTableService = ctx.shardingTableService; - this.networkModuleManager = ctx.networkModuleManager; this.errorType = ERROR_TYPE.COMMIT_PROOF.SUBMIT_UPDATE_COMMIT_ERROR; } @@ -30,6 +27,14 @@ class SubmitUpdateCommitCommand extends Command { operationId, } = command.data; + this.logger.trace( + `Started ${command.name} for agreement id: ${agreementId} ` + + `blockchain: ${blockchain} contract: ${contract}, token id: ${tokenId}, ` + + `keyword: ${keyword}, hash function id: ${hashFunctionId}. Retry number ${ + COMMAND_RETRIES.SUBMIT_UPDATE_COMMIT - command.retries + 1 + }`, + ); + const epoch = await this.calculateCurrentEpoch( Number(agreementData.startTime), Number(agreementData.epochLength), @@ -45,14 +50,6 @@ class SubmitUpdateCommitCommand extends Command { ); } - this.logger.trace( - `Started ${command.name} for agreement id: ${command.data.agreementId} ` + - `blockchain: ${blockchain} contract: ${contract}, token id: ${tokenId}, ` + - `keyword: ${keyword}, hash function id: ${hashFunctionId}. Retry number ${ - COMMAND_RETRIES.SUBMIT_UPDATE_COMMIT - command.retries + 1 - }`, - ); - const hasPendingUpdate = await this.blockchainModuleManager.hasPendingUpdate( blockchain, tokenId, @@ -105,6 +102,16 @@ class SubmitUpdateCommitCommand extends Command { }, ); + const transactionQueueLength = + this.blockchainModuleManager.getTransactionQueueLength(blockchain); + + this.logger.trace( + `Scheduled submit update commit transaction for agreement id: ${agreementId} ` + + `blockchain: ${blockchain} contract: ${contract}, token id: ${tokenId}, ` + + `keyword: ${keyword}, hash function id: ${hashFunctionId}, operationId ${operationId}` + + `transaction queue length: ${transactionQueueLength}.`, + ); + return Command.empty(); } diff --git a/src/commands/protocols/update/receiver/v1.0.0/v1-0-0-handle-update-request-command.js b/src/commands/protocols/update/receiver/v1.0.0/v1-0-0-handle-update-request-command.js index 717a4d999..5efddae7f 100644 --- a/src/commands/protocols/update/receiver/v1.0.0/v1-0-0-handle-update-request-command.js +++ b/src/commands/protocols/update/receiver/v1.0.0/v1-0-0-handle-update-request-command.js @@ -60,23 +60,42 @@ class HandleUpdateRequestCommand extends HandleProtocolMessageCommand { const updateCommitWindowDuration = await this.blockchainModuleManager.getUpdateCommitWindowDuration(blockchain); - const R0 = await this.blockchainModuleManager.getR0(blockchain); - const R2 = await this.blockchainModuleManager.getR2(blockchain); - - const rank = await this.calculateRank(blockchain, keyword, hashFunctionId, R2); - this.logger.trace(`Calculated rank: ${rank + 1} for agreement id: ${agreementId}`); - const finalizationCommitsNumber = - await this.blockchainModuleManager.getFinalizationCommitsNumber(blockchain); - - const updateCommitDelay = await this.calculateUpdateCommitDelay( - blockchain, - updateCommitWindowDuration, - finalizationCommitsNumber, - R0, - rank, - ); - - await Promise.all([ + const r0 = await this.blockchainModuleManager.getR0(blockchain); + const r2 = await this.blockchainModuleManager.getR2(blockchain); + const scheduleCommandsPromises = []; + + const rank = await this.calculateRank(blockchain, keyword, hashFunctionId, r2); + if (rank != null) { + this.logger.trace(`Calculated rank: ${rank + 1} for agreement id: ${agreementId}`); + const finalizationCommitsNumber = + await this.blockchainModuleManager.getFinalizationCommitsNumber(blockchain); + + const updateCommitDelay = await this.calculateUpdateCommitDelay( + blockchain, + updateCommitWindowDuration, + finalizationCommitsNumber, + r0, + rank, + ); + scheduleCommandsPromises.push( + this.commandExecutor.add({ + name: 'submitUpdateCommitCommand', + delay: updateCommitDelay, + retries: COMMAND_RETRIES.SUBMIT_UPDATE_COMMIT, + data: { + ...commandData, + agreementData, + agreementId, + r0, + r2, + updateCommitWindowDuration, + }, + transactional: false, + }), + ); + } + + scheduleCommandsPromises.push( this.commandExecutor.add({ name: 'deletePendingStateCommand', sequence: [], @@ -84,22 +103,9 @@ class HandleUpdateRequestCommand extends HandleProtocolMessageCommand { data: { ...commandData, repository: PENDING_STORAGE_REPOSITORIES.PUBLIC }, transactional: false, }), - this.commandExecutor.add({ - name: 'submitUpdateCommitCommand', - delay: updateCommitDelay, - retries: COMMAND_RETRIES.SUBMIT_UPDATE_COMMIT, - data: { - ...commandData, - agreementData, - agreementId, - R0, - R2, - rank, - updateCommitWindowDuration, - }, - transactional: false, - }), - ]); + ); + + await Promise.all(scheduleCommandsPromises); await this.operationIdService.updateOperationIdStatus( operationId, OPERATION_ID_STATUS.UPDATE.VALIDATING_UPDATE_ASSERTION_REMOTE_END, @@ -111,7 +117,7 @@ class HandleUpdateRequestCommand extends HandleProtocolMessageCommand { blockchain, updateCommitWindowDuration, finalizationCommitsNumber, - R0, + r0, rank, ) { const r0OffsetPeriod = 0; @@ -130,21 +136,26 @@ class HandleUpdateRequestCommand extends HandleProtocolMessageCommand { this.logger.info( `Calculated update commit delay: ${Math.floor( delay / 1000, - )}s, commitsBlockDuration: ${commitsBlockDuration}, commitBlock: ${commitBlock}, r0OffsetPeriod:${r0OffsetPeriod}, updateCommitWindowDuration ${updateCommitWindowDuration}s, finalizationCommitsNumber: ${finalizationCommitsNumber}, r0: ${R0}, rank: ${rank}`, + )}s, commitsBlockDuration: ${commitsBlockDuration}, commitBlock: ${commitBlock}, r0OffsetPeriod:${r0OffsetPeriod}, updateCommitWindowDuration ${updateCommitWindowDuration}s, finalizationCommitsNumber: ${finalizationCommitsNumber}, r0: ${r0}, rank: ${rank}`, ); return delay; } - async calculateRank(blockchain, keyword, hashFunctionId, R2) { + async calculateRank(blockchain, keyword, hashFunctionId, r2) { const neighbourhood = await this.shardingTableService.findNeighbourhood( blockchain, keyword, - R2, + r2, hashFunctionId, true, ); + const peerId = this.networkModuleManager.getPeerId().toB58String(); + if (!neighbourhood.some((node) => node.peer_id === peerId)) { + return; + } + const scores = await Promise.all( neighbourhood.map(async (node) => ({ score: await this.serviceAgreementService.calculateScore( @@ -159,9 +170,7 @@ class HandleUpdateRequestCommand extends HandleProtocolMessageCommand { scores.sort((a, b) => b.score - a.score); - return scores.findIndex( - (node) => node.peerId === this.networkModuleManager.getPeerId().toB58String(), - ); + return scores.findIndex((node) => node.peerId === peerId); } /** diff --git a/src/modules/blockchain/blockchain-module-manager.js b/src/modules/blockchain/blockchain-module-manager.js index b571962a8..182108b11 100644 --- a/src/modules/blockchain/blockchain-module-manager.js +++ b/src/modules/blockchain/blockchain-module-manager.js @@ -5,6 +5,10 @@ class BlockchainModuleManager extends BaseModuleManager { return 'blockchain'; } + getTransactionQueueLength(blockchain) { + return this.callImplementationFunction(blockchain, 'getTransactionQueueLength'); + } + async initializeContracts(blockchain) { return this.callImplementationFunction(blockchain, 'initializeContracts'); } diff --git a/src/modules/blockchain/implementation/web3-service.js b/src/modules/blockchain/implementation/web3-service.js index 65d900dfd..42e4c866b 100644 --- a/src/modules/blockchain/implementation/web3-service.js +++ b/src/modules/blockchain/implementation/web3-service.js @@ -81,6 +81,10 @@ class Web3Service { ); } + getTransactionQueueLength() { + return this.transactionQueue.length(); + } + async initializeWeb3() { let tries = 0; let isRpcConnected = false; @@ -356,7 +360,7 @@ class Web3Service { this.logger.info( 'Sending signed transaction to blockchain, calling method: ' + - `${functionName} with gas limit: ${gas.toString()} and gasPrice ${gasPrice.toString()}`, + `${functionName} with gas limit: ${gas.toString()} and gasPrice ${gasPrice.toString()}. Transaction queue length: ${this.getTransactionQueueLength()}`, ); const tx = await contractInstance[functionName](...args, { gasPrice, From 28110af765a9ab119155459b5531dee11489fdc5 Mon Sep 17 00:00:00 2001 From: zeroxbt <89495162+zeroxbt@users.noreply.github.com> Date: Wed, 17 May 2023 15:36:57 +0200 Subject: [PATCH 09/16] bump version to 6.0.9 (#2533) --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 433b657a4..7d1bc7066 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "origintrail_node", - "version": "6.0.9+hotfix.1", + "version": "6.0.9", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "origintrail_node", - "version": "6.0.9+hotfix.1", + "version": "6.0.9", "license": "ISC", "dependencies": { "@comunica/query-sparql": "^2.4.3", diff --git a/package.json b/package.json index 86e12dbd6..9a9ee6a17 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "origintrail_node", - "version": "6.0.9+hotfix.1", + "version": "6.0.9", "description": "OTNode V6", "main": "index.js", "type": "module", From 2d77233d82236a7db18ed3fba773b67cde4ddfdc Mon Sep 17 00:00:00 2001 From: zeroxbt <89495162+zeroxbt@users.noreply.github.com> Date: Wed, 17 May 2023 15:54:30 +0200 Subject: [PATCH 10/16] remove snake case variables (#2530) --- .eslintrc.cjs | 3 +- src/commands/command-executor.js | 30 +-- src/commands/common/dial-peers-command.js | 2 +- .../protocols/common/epoch-check-command.js | 56 ++-- .../protocols/common/find-nodes-command.js | 4 +- .../common/handle-protocol-message-command.js | 5 +- .../v1-0-0-handle-update-request-command.js | 4 +- .../pull-sharding-table-migration.js | 4 +- .../sequelize/models/ability.js | 4 +- .../sequelize/models/blockchain-event.js | 4 +- .../sequelize/models/blockchain.js | 6 +- .../sequelize/models/commands.js | 13 +- .../implementation/sequelize/models/event.js | 8 +- .../sequelize/models/get-response.js | 14 +- .../implementation/sequelize/models/get.js | 8 +- .../sequelize/models/operation_ids.js | 14 +- .../sequelize/models/publish-response.js | 14 +- .../sequelize/models/publish.js | 8 +- .../sequelize/models/role-ability.js | 4 +- .../implementation/sequelize/models/role.js | 4 +- .../sequelize/models/service-agreement.js | 30 +-- .../implementation/sequelize/models/shard.js | 8 +- .../implementation/sequelize/models/token.js | 8 +- .../sequelize/models/update-response.js | 14 +- .../implementation/sequelize/models/update.js | 8 +- .../implementation/sequelize/models/user.js | 4 +- .../blockchain-event-repository.js | 4 +- .../repositories/blockchain-repository.js | 10 +- .../repositories/command-repository.js | 2 +- .../repositories/event-repository.js | 4 +- .../repositories/operation-id-repository.js | 4 +- .../repositories/operation-repository.js | 6 +- .../repositories/operation-response.js | 4 +- .../service-agreement-repository.js | 80 +++--- .../repositories/shard-repository.js | 53 ++-- .../repositories/token-repository.js | 4 +- .../blockchain-event-listener-service.js | 53 ++-- src/service/operation-id-service.js | 2 +- src/service/sharding-table-service.js | 8 +- test/modules/repository/repository.js | 245 +++++++++--------- .../mock/repository-module-manager-mock.js | 102 ++++---- 41 files changed, 426 insertions(+), 436 deletions(-) diff --git a/.eslintrc.cjs b/.eslintrc.cjs index 6602f7585..32d9cc8d0 100644 --- a/.eslintrc.cjs +++ b/.eslintrc.cjs @@ -10,13 +10,12 @@ module.exports = { }, rules: { 'linebreak-style': ['error', 'unix'], - camelcase: 0, 'class-methods-use-this': 0, 'consistent-return': 0, 'no-restricted-syntax': 0, 'guard-for-in': 0, 'no-console': 'warn', - 'no-continue': 1, + 'no-continue': 0, 'no-underscore-dangle': 0, 'import/extensions': 0, }, diff --git a/src/commands/command-executor.js b/src/commands/command-executor.js index 3e0428c81..963cef3fd 100644 --- a/src/commands/command-executor.js +++ b/src/commands/command-executor.js @@ -75,7 +75,7 @@ class CommandExecutor { const command = executeCommand; const now = Date.now(); await this._update(command, { - started_at: now, + startedAt: now, }); if (this.verboseLoggingEnabled) { @@ -90,7 +90,7 @@ class CommandExecutor { }); return; } - if (command.deadline_at && now > command.deadline_at) { + if (command.deadlineAt && now > command.deadlineAt) { this.logger.warn(`Command ${command.name} and ID ${command.id} is too late...`); await this._update(command, { status: COMMAND_STATUS.EXPIRED, @@ -108,7 +108,7 @@ class CommandExecutor { return; } - const waitMs = command.ready_at + command.delay - now; + const waitMs = command.readyAt + command.delay - now; if (waitMs > 0) { if (this.verboseLoggingEnabled) { this.logger.trace( @@ -158,7 +158,7 @@ class CommandExecutor { const children = result.commands.map((c) => { const newCommand = c; - newCommand.parent_id = command.id; + newCommand.parentId = command.id; return newCommand; }); @@ -250,10 +250,10 @@ class CommandExecutor { const now = Date.now(); if (delay != null && delay > MAX_COMMAND_DELAY_IN_MILLS) { - if (command.ready_at == null) { - command.ready_at = now; + if (command.readyAt == null) { + command.readyAt = now; } - command.ready_at += delay; + command.readyAt += delay; delay = MAX_COMMAND_DELAY_IN_MILLS; } @@ -339,8 +339,8 @@ class CommandExecutor { [command.name] = command.sequence; command.sequence = command.sequence.slice(1); } - if (!command.ready_at) { - command.ready_at = Date.now(); // take current time + if (!command.readyAt) { + command.readyAt = Date.now(); // take current time } if (command.delay == null) { command.delay = 0; @@ -414,10 +414,10 @@ class CommandExecutor { // TODO consider JOIN instead const commands = pendingCommands.filter(async (pc) => { - if (!pc.parent_id) { + if (!pc.parentId) { return true; } - const parent = await this.repositoryModuleManager.getCommandWithId(pc.parent_id); + const parent = await this.repositoryModuleManager.getCommandWithId(pc.parentId); return !parent || parent.status === 'COMPLETED'; }); @@ -427,14 +427,14 @@ class CommandExecutor { id: commandModel.id, name: commandModel.name, data: commandModel.data, - ready_at: commandModel.ready_at, + readyAt: commandModel.readyAt, delay: commandModel.delay, - started_at: commandModel.started_at, - deadline_at: commandModel.deadline_at, + startedAt: commandModel.startedAt, + deadlineAt: commandModel.deadlineAt, period: commandModel.period, status: commandModel.status, message: commandModel.message, - parent_id: commandModel.parent_id, + parentId: commandModel.parentId, transactional: commandModel.transactional, retries: commandModel.retries, sequence: commandModel.sequence, diff --git a/src/commands/common/dial-peers-command.js b/src/commands/common/dial-peers-command.js index 0afd86007..41a728f34 100644 --- a/src/commands/common/dial-peers-command.js +++ b/src/commands/common/dial-peers-command.js @@ -25,7 +25,7 @@ class DialPeersCommand extends Command { if (peersToDial.length) { this.logger.trace(`Dialing ${peersToDial.length} remote peers`); await Promise.all( - peersToDial.map(({ peer_id: peerId }) => this.shardingTableService.dial(peerId)), + peersToDial.map(({ peerId }) => this.shardingTableService.dial(peerId)), ); } diff --git a/src/commands/protocols/common/epoch-check-command.js b/src/commands/protocols/common/epoch-check-command.js index 91a3381b1..552120120 100644 --- a/src/commands/protocols/common/epoch-check-command.js +++ b/src/commands/protocols/common/epoch-check-command.js @@ -63,7 +63,7 @@ class EpochCheckCommand extends Command { const rank = await this.calculateRank( blockchain, serviceAgreement.keyword, - serviceAgreement.hash_function_id, + serviceAgreement.hashFunctionId, r2, ); @@ -76,7 +76,7 @@ class EpochCheckCommand extends Command { if (rank == null) { this.logger.trace( - `Node not in R2: ${r2} for agreement id: ${serviceAgreement.agreement_id}. Skipping scheduling submit commit command-`, + `Node not in R2: ${r2} for agreement id: ${serviceAgreement.agreementId}. Skipping scheduling submit commit command.`, ); continue; } @@ -84,7 +84,7 @@ class EpochCheckCommand extends Command { if (rank >= r0) { this.logger.trace( `Calculated rank: ${rank + 1}. Node not in R0: ${r0} for agreement id: ${ - serviceAgreement.agreement_id + serviceAgreement.agreementId }. Skipping scheduling submit commit command.`, ); continue; @@ -92,7 +92,7 @@ class EpochCheckCommand extends Command { this.logger.trace( `Calculated rank: ${rank + 1}. Node in R0: ${r0} for agreement id: ${ - serviceAgreement.agreement_id + serviceAgreement.agreementId }. Scheduling submit commit command.`, ); @@ -119,13 +119,13 @@ class EpochCheckCommand extends Command { const eligibleForReward = await this.isEligibleForRewards( blockchain, - serviceAgreement.agreement_id, - serviceAgreement.current_epoch, - serviceAgreement.state_index, + serviceAgreement.agreementId, + serviceAgreement.currentEpoch, + serviceAgreement.stateIndex, ); if (eligibleForReward) { this.logger.trace( - `Node is eligible for rewards for agreement id: ${serviceAgreement.agreement_id}. Scheduling submit proof command.`, + `Node is eligible for rewards for agreement id: ${serviceAgreement.agreementId}. Scheduling submit proof command.`, ); scheduleSubmitProofCommands.push( @@ -133,13 +133,13 @@ class EpochCheckCommand extends Command { ); } else { this.logger.trace( - `Node is not eligible for rewards for agreement id: ${serviceAgreement.agreement_id}. Skipping scheduling submit proof command.`, + `Node is not eligible for rewards for agreement id: ${serviceAgreement.agreementId}. Skipping scheduling submit proof command.`, ); } updateServiceAgreementsLastProofEpoch.push( this.repositoryModuleManager.updateServiceAgreementLastProofEpoch( - serviceAgreement.agreement_id, - serviceAgreement.current_epoch, + serviceAgreement.agreementId, + serviceAgreement.currentEpoch, ), ); } @@ -166,12 +166,12 @@ class EpochCheckCommand extends Command { const scores = await Promise.all( neighbourhood.map(async (node) => ({ score: await this.serviceAgreementService.calculateScore( - node.peer_id, + node.peerId, blockchain, keyword, hashFunctionId, ), - peerId: node.peer_id, + peerId: node.peerId, })), ); @@ -201,14 +201,14 @@ class EpochCheckCommand extends Command { async scheduleSubmitCommitCommand(agreement) { const commandData = { - blockchain: agreement.blockchain_id, - contract: agreement.asset_storage_contract_address, - tokenId: agreement.token_id, + blockchain: agreement.blockchainId, + contract: agreement.assetStorageContractAddress, + tokenId: agreement.tokenId, keyword: agreement.keyword, - hashFunctionId: agreement.hash_function_id, - epoch: agreement.current_epoch, - agreementId: agreement.agreement_id, - stateIndex: agreement.state_index, + hashFunctionId: agreement.hashFunctionId, + epoch: agreement.currentEpoch, + agreementId: agreement.agreementId, + stateIndex: agreement.stateIndex, }; await this.commandExecutor.add({ @@ -222,15 +222,15 @@ class EpochCheckCommand extends Command { async scheduleSubmitProofsCommand(agreement) { const commandData = { - blockchain: agreement.blockchain_id, - contract: agreement.asset_storage_contract_address, - tokenId: agreement.token_id, + blockchain: agreement.blockchainId, + contract: agreement.assetStorageContractAddress, + tokenId: agreement.tokenId, keyword: agreement.keyword, - hashFunctionId: agreement.hash_function_id, - epoch: agreement.current_epoch, - agreementId: agreement.agreement_id, - assertionId: agreement.assertion_id, - stateIndex: agreement.state_index, + hashFunctionId: agreement.hashFunctionId, + epoch: agreement.currentEpoch, + agreementId: agreement.agreementId, + assertionId: agreement.assertionId, + stateIndex: agreement.stateIndex, }; return this.commandExecutor.add({ diff --git a/src/commands/protocols/common/find-nodes-command.js b/src/commands/protocols/common/find-nodes-command.js index fac4b2d3e..400a0a6ef 100644 --- a/src/commands/protocols/common/find-nodes-command.js +++ b/src/commands/protocols/common/find-nodes-command.js @@ -80,8 +80,8 @@ class FindNodesCommand extends Command { ); const nodesFound = await Promise.all( - closestNodes.map((node) => - this.shardingTableService.findPeerAddressAndProtocols(node.peer_id), + closestNodes.map(({ peerId }) => + this.shardingTableService.findPeerAddressAndProtocols(peerId), ), ); diff --git a/src/commands/protocols/common/handle-protocol-message-command.js b/src/commands/protocols/common/handle-protocol-message-command.js index 0bdbc46be..36e560dcc 100644 --- a/src/commands/protocols/common/handle-protocol-message-command.js +++ b/src/commands/protocols/common/handle-protocol-message-command.js @@ -53,8 +53,9 @@ class HandleProtocolMessageCommand extends Command { hashFunctionId, true, ); - for (const { peer_id } of closestNodes) { - if (peer_id === this.networkModuleManager.getPeerId().toB58String()) { + const peerId = this.networkModuleManager.getPeerId().toB58String(); + for (const { peerId: otherPeerId } of closestNodes) { + if (otherPeerId === peerId) { return true; } } diff --git a/src/commands/protocols/update/receiver/v1.0.0/v1-0-0-handle-update-request-command.js b/src/commands/protocols/update/receiver/v1.0.0/v1-0-0-handle-update-request-command.js index 5efddae7f..380ecdd76 100644 --- a/src/commands/protocols/update/receiver/v1.0.0/v1-0-0-handle-update-request-command.js +++ b/src/commands/protocols/update/receiver/v1.0.0/v1-0-0-handle-update-request-command.js @@ -159,12 +159,12 @@ class HandleUpdateRequestCommand extends HandleProtocolMessageCommand { const scores = await Promise.all( neighbourhood.map(async (node) => ({ score: await this.serviceAgreementService.calculateScore( - node.peer_id, + node.peerId, blockchain, keyword, hashFunctionId, ), - peerId: node.peer_id, + peerId: node.peerId, })), ); diff --git a/src/migration/pull-sharding-table-migration.js b/src/migration/pull-sharding-table-migration.js index b1150a35e..6a02e219f 100644 --- a/src/migration/pull-sharding-table-migration.js +++ b/src/migration/pull-sharding-table-migration.js @@ -65,8 +65,8 @@ class PullBlockchainShardingTableMigration extends BaseMigration { ); return { - peer_id: nodeId, - blockchain_id: blockchainId, + peerId: nodeId, + blockchainId, ask: this.blockchainModuleManager.convertFromWei( blockchainId, peer.ask, diff --git a/src/modules/repository/implementation/sequelize/models/ability.js b/src/modules/repository/implementation/sequelize/models/ability.js index f58bf19b8..ebc382c7c 100644 --- a/src/modules/repository/implementation/sequelize/models/ability.js +++ b/src/modules/repository/implementation/sequelize/models/ability.js @@ -3,8 +3,8 @@ export default (sequelize, DataTypes) => { 'ability', { name: DataTypes.STRING, - created_at: DataTypes.DATE, - updated_at: DataTypes.DATE, + createdAt: DataTypes.DATE, + updatedAt: DataTypes.DATE, }, { underscored: true }, ); diff --git a/src/modules/repository/implementation/sequelize/models/blockchain-event.js b/src/modules/repository/implementation/sequelize/models/blockchain-event.js index fa00561d5..c555dd1b8 100644 --- a/src/modules/repository/implementation/sequelize/models/blockchain-event.js +++ b/src/modules/repository/implementation/sequelize/models/blockchain-event.js @@ -8,13 +8,13 @@ export default (sequelize, DataTypes) => { autoIncrement: true, }, contract: DataTypes.STRING, - blockchain_id: DataTypes.STRING, + blockchainId: DataTypes.STRING, event: DataTypes.STRING, data: DataTypes.TEXT, block: DataTypes.INTEGER, processed: DataTypes.BOOLEAN, }, - {}, + { underscored: true }, ); event.associate = () => { // associations can be defined here diff --git a/src/modules/repository/implementation/sequelize/models/blockchain.js b/src/modules/repository/implementation/sequelize/models/blockchain.js index ea3451f40..6bb797780 100644 --- a/src/modules/repository/implementation/sequelize/models/blockchain.js +++ b/src/modules/repository/implementation/sequelize/models/blockchain.js @@ -2,7 +2,7 @@ export default (sequelize, DataTypes) => { const blockchain = sequelize.define( 'blockchain', { - blockchain_id: { + blockchainId: { type: DataTypes.STRING, primaryKey: true, }, @@ -10,8 +10,8 @@ export default (sequelize, DataTypes) => { type: DataTypes.STRING, primaryKey: true, }, - last_checked_block: DataTypes.BIGINT, - last_checked_timestamp: DataTypes.BIGINT, + lastCheckedBlock: DataTypes.BIGINT, + lastCheckedTimestamp: DataTypes.BIGINT, }, { underscored: true }, ); diff --git a/src/modules/repository/implementation/sequelize/models/commands.js b/src/modules/repository/implementation/sequelize/models/commands.js index e0ec17b0a..dde59ba7f 100644 --- a/src/modules/repository/implementation/sequelize/models/commands.js +++ b/src/modules/repository/implementation/sequelize/models/commands.js @@ -18,26 +18,27 @@ export default (sequelize, DataTypes) => { name: DataTypes.STRING, data: DataTypes.JSON, sequence: DataTypes.JSON, - ready_at: DataTypes.BIGINT, + readyAt: DataTypes.BIGINT, delay: DataTypes.BIGINT, - started_at: DataTypes.BIGINT, - deadline_at: DataTypes.BIGINT, + startedAt: DataTypes.BIGINT, + deadlineAt: DataTypes.BIGINT, period: DataTypes.INTEGER, status: DataTypes.STRING, message: DataTypes.TEXT, - parent_id: DataTypes.UUID, + parentId: DataTypes.UUID, transactional: DataTypes.BOOLEAN, retries: { type: DataTypes.INTEGER, defaultValue: 0, }, - created_at: DataTypes.DATE, - updated_at: DataTypes.DATE, + createdAt: DataTypes.DATE, + updatedAt: DataTypes.DATE, }, { sequelize, modelName: 'commands', timestamps: false, + underscored: true, }, ); return commands; diff --git a/src/modules/repository/implementation/sequelize/models/event.js b/src/modules/repository/implementation/sequelize/models/event.js index 360da526d..77de650a7 100644 --- a/src/modules/repository/implementation/sequelize/models/event.js +++ b/src/modules/repository/implementation/sequelize/models/event.js @@ -7,16 +7,16 @@ export default (sequelize, DataTypes) => { primaryKey: true, autoIncrement: true, }, - operation_id: DataTypes.UUID, + operationId: DataTypes.UUID, name: DataTypes.STRING, timestamp: DataTypes.STRING, value1: DataTypes.TEXT, value2: DataTypes.TEXT, value3: DataTypes.TEXT, - created_at: DataTypes.DATE, - updated_at: DataTypes.DATE, + createdAt: DataTypes.DATE, + updatedAt: DataTypes.DATE, }, - {}, + { underscored: true }, ); event.associate = () => { // associations can be defined here diff --git a/src/modules/repository/implementation/sequelize/models/get-response.js b/src/modules/repository/implementation/sequelize/models/get-response.js index a1e62ef56..201873eb8 100644 --- a/src/modules/repository/implementation/sequelize/models/get-response.js +++ b/src/modules/repository/implementation/sequelize/models/get-response.js @@ -1,5 +1,5 @@ export default (sequelize, DataTypes) => { - const get_response = sequelize.define( + const getResponse = sequelize.define( 'get_response', { id: { @@ -7,17 +7,17 @@ export default (sequelize, DataTypes) => { primaryKey: true, autoIncrement: true, }, - operation_id: DataTypes.UUID, + operationId: DataTypes.UUID, keyword: DataTypes.STRING, status: DataTypes.STRING, message: DataTypes.TEXT, - created_at: DataTypes.DATE, - updated_at: DataTypes.DATE, + createdAt: DataTypes.DATE, + updatedAt: DataTypes.DATE, }, - {}, + { underscored: true }, ); - get_response.associate = () => { + getResponse.associate = () => { // associations can be defined here }; - return get_response; + return getResponse; }; diff --git a/src/modules/repository/implementation/sequelize/models/get.js b/src/modules/repository/implementation/sequelize/models/get.js index 6c49d8a46..7c9cdad37 100644 --- a/src/modules/repository/implementation/sequelize/models/get.js +++ b/src/modules/repository/implementation/sequelize/models/get.js @@ -7,12 +7,12 @@ export default (sequelize, DataTypes) => { primaryKey: true, autoIncrement: true, }, - operation_id: DataTypes.UUID, + operationId: DataTypes.UUID, status: DataTypes.STRING, - created_at: DataTypes.DATE, - updated_at: DataTypes.DATE, + createdAt: DataTypes.DATE, + updatedAt: DataTypes.DATE, }, - {}, + { underscored: true }, ); get.associate = () => { // associations can be defined here diff --git a/src/modules/repository/implementation/sequelize/models/operation_ids.js b/src/modules/repository/implementation/sequelize/models/operation_ids.js index 4c42daab7..f26d85f5d 100644 --- a/src/modules/repository/implementation/sequelize/models/operation_ids.js +++ b/src/modules/repository/implementation/sequelize/models/operation_ids.js @@ -1,10 +1,10 @@ import { v4 as uuidv4 } from 'uuid'; export default (sequelize, DataTypes) => { - const operation_ids = sequelize.define( + const operationIds = sequelize.define( 'operation_ids', { - operation_id: { + operationId: { type: DataTypes.UUID, primaryKey: true, defaultValue: () => uuidv4(), @@ -15,13 +15,13 @@ export default (sequelize, DataTypes) => { type: DataTypes.BIGINT, defaultValue: () => Date.now(), }, - created_at: DataTypes.DATE, - updated_at: DataTypes.DATE, + createdAt: DataTypes.DATE, + updatedAt: DataTypes.DATE, }, - {}, + { underscored: true }, ); - operation_ids.associate = () => { + operationIds.associate = () => { // associations can be defined here }; - return operation_ids; + return operationIds; }; diff --git a/src/modules/repository/implementation/sequelize/models/publish-response.js b/src/modules/repository/implementation/sequelize/models/publish-response.js index 1289df773..f9b44bc5e 100644 --- a/src/modules/repository/implementation/sequelize/models/publish-response.js +++ b/src/modules/repository/implementation/sequelize/models/publish-response.js @@ -1,5 +1,5 @@ export default (sequelize, DataTypes) => { - const publish_response = sequelize.define( + const publishResponse = sequelize.define( 'publish_response', { id: { @@ -7,17 +7,17 @@ export default (sequelize, DataTypes) => { primaryKey: true, autoIncrement: true, }, - operation_id: DataTypes.UUID, + operationId: DataTypes.UUID, keyword: DataTypes.STRING, status: DataTypes.STRING, message: DataTypes.TEXT, - created_at: DataTypes.DATE, - updated_at: DataTypes.DATE, + createdAt: DataTypes.DATE, + updatedAt: DataTypes.DATE, }, - {}, + { underscored: true }, ); - publish_response.associate = () => { + publishResponse.associate = () => { // associations can be defined here }; - return publish_response; + return publishResponse; }; diff --git a/src/modules/repository/implementation/sequelize/models/publish.js b/src/modules/repository/implementation/sequelize/models/publish.js index 1dcfe2178..9957cfd60 100644 --- a/src/modules/repository/implementation/sequelize/models/publish.js +++ b/src/modules/repository/implementation/sequelize/models/publish.js @@ -7,12 +7,12 @@ export default (sequelize, DataTypes) => { primaryKey: true, autoIncrement: true, }, - operation_id: DataTypes.UUID, + operationId: DataTypes.UUID, status: DataTypes.STRING, - created_at: DataTypes.DATE, - updated_at: DataTypes.DATE, + createdAt: DataTypes.DATE, + updatedAt: DataTypes.DATE, }, - {}, + { underscored: true }, ); publish.associate = () => { // associations can be defined here diff --git a/src/modules/repository/implementation/sequelize/models/role-ability.js b/src/modules/repository/implementation/sequelize/models/role-ability.js index 36720e090..4417e4964 100644 --- a/src/modules/repository/implementation/sequelize/models/role-ability.js +++ b/src/modules/repository/implementation/sequelize/models/role-ability.js @@ -2,8 +2,8 @@ export default (sequelize, DataTypes) => { const roleAbility = sequelize.define( 'role_ability', { - created_at: DataTypes.DATE, - updated_at: DataTypes.DATE, + createdAt: DataTypes.DATE, + updatedAt: DataTypes.DATE, }, { underscored: true }, ); diff --git a/src/modules/repository/implementation/sequelize/models/role.js b/src/modules/repository/implementation/sequelize/models/role.js index 4b42d42ce..432adcb37 100644 --- a/src/modules/repository/implementation/sequelize/models/role.js +++ b/src/modules/repository/implementation/sequelize/models/role.js @@ -3,8 +3,8 @@ export default (sequelize, DataTypes) => { 'role', { name: DataTypes.STRING, - created_at: DataTypes.DATE, - updated_at: DataTypes.DATE, + createdAt: DataTypes.DATE, + updatedAt: DataTypes.DATE, }, { underscored: true }, ); diff --git a/src/modules/repository/implementation/sequelize/models/service-agreement.js b/src/modules/repository/implementation/sequelize/models/service-agreement.js index 52f6707c7..e0bc30fd9 100644 --- a/src/modules/repository/implementation/sequelize/models/service-agreement.js +++ b/src/modules/repository/implementation/sequelize/models/service-agreement.js @@ -2,47 +2,47 @@ export default (sequelize, DataTypes) => { const serviceAgreement = sequelize.define( 'service_agreement', { - blockchain_id: { + blockchainId: { type: DataTypes.STRING, allowNull: false, }, - asset_storage_contract_address: { + assetStorageContractAddress: { type: DataTypes.STRING(42), allowNull: false, }, - token_id: { + tokenId: { type: DataTypes.INTEGER.UNSIGNED, allowNull: false, }, - agreement_id: { + agreementId: { type: DataTypes.STRING, primaryKey: true, }, - start_time: { + startTime: { type: DataTypes.INTEGER.UNSIGNED, allowNull: false, }, - epochs_number: { + epochsNumber: { type: DataTypes.SMALLINT.UNSIGNED, allowNull: false, }, - epoch_length: { + epochLength: { type: DataTypes.INTEGER.UNSIGNED, allowNull: false, }, - score_function_id: { + scoreFunctionId: { type: DataTypes.TINYINT.UNSIGNED, allowNull: false, }, - state_index: { + stateIndex: { type: DataTypes.SMALLINT.UNSIGNED, allowNull: false, }, - assertion_id: { + assertionId: { type: DataTypes.STRING, allowNull: false, }, - hash_function_id: { + hashFunctionId: { type: DataTypes.TINYINT.UNSIGNED, allowNull: false, }, @@ -50,18 +50,18 @@ export default (sequelize, DataTypes) => { type: DataTypes.STRING, allowNull: false, }, - proof_window_offset_perc: { + proofWindowOffsetPerc: { type: DataTypes.TINYINT.UNSIGNED, allowNull: false, }, - last_commit_epoch: { + lastCommitEpoch: { type: DataTypes.SMALLINT.UNSIGNED, }, - last_proof_epoch: { + lastProofEpoch: { type: DataTypes.SMALLINT.UNSIGNED, }, }, - {}, + { underscored: true }, ); serviceAgreement.associate = () => { // associations can be defined here diff --git a/src/modules/repository/implementation/sequelize/models/shard.js b/src/modules/repository/implementation/sequelize/models/shard.js index 8d94ae5d7..cd14aa539 100644 --- a/src/modules/repository/implementation/sequelize/models/shard.js +++ b/src/modules/repository/implementation/sequelize/models/shard.js @@ -2,8 +2,8 @@ export default (sequelize, DataTypes) => { const shard = sequelize.define( 'shard', { - peer_id: { type: DataTypes.STRING, primaryKey: true }, - blockchain_id: { type: DataTypes.STRING, primaryKey: true }, + peerId: { type: DataTypes.STRING, primaryKey: true }, + blockchainId: { type: DataTypes.STRING, primaryKey: true }, ask: { type: DataTypes.INTEGER, allowNull: false, @@ -12,12 +12,12 @@ export default (sequelize, DataTypes) => { type: DataTypes.INTEGER, allowNull: false, }, - last_seen: { + lastSeen: { type: DataTypes.DATE, allowNull: false, defaultValue: new Date(0), }, - last_dialed: { + lastDialed: { type: DataTypes.DATE, allowNull: false, defaultValue: new Date(0), diff --git a/src/modules/repository/implementation/sequelize/models/token.js b/src/modules/repository/implementation/sequelize/models/token.js index d3d5a4be1..f447c97e9 100644 --- a/src/modules/repository/implementation/sequelize/models/token.js +++ b/src/modules/repository/implementation/sequelize/models/token.js @@ -4,13 +4,13 @@ export default (sequelize, DataTypes) => { { id: { type: DataTypes.STRING, primaryKey: true }, revoked: DataTypes.BOOLEAN, - user_id: DataTypes.INTEGER, + userId: DataTypes.INTEGER, name: { type: DataTypes.STRING, }, - expires_at: DataTypes.DATE, - created_at: DataTypes.DATE, - updated_at: DataTypes.DATE, + expiresAt: DataTypes.DATE, + createdAt: DataTypes.DATE, + updatedAt: DataTypes.DATE, }, { underscored: true }, ); diff --git a/src/modules/repository/implementation/sequelize/models/update-response.js b/src/modules/repository/implementation/sequelize/models/update-response.js index 19d98647b..40c6143c8 100644 --- a/src/modules/repository/implementation/sequelize/models/update-response.js +++ b/src/modules/repository/implementation/sequelize/models/update-response.js @@ -1,5 +1,5 @@ export default (sequelize, DataTypes) => { - const update_response = sequelize.define( + const updateResponse = sequelize.define( 'update_response', { id: { @@ -7,17 +7,17 @@ export default (sequelize, DataTypes) => { primaryKey: true, autoIncrement: true, }, - operation_id: DataTypes.UUID, + operationId: DataTypes.UUID, keyword: DataTypes.STRING, status: DataTypes.STRING, message: DataTypes.TEXT, - created_at: DataTypes.DATE, - updated_at: DataTypes.DATE, + createdAt: DataTypes.DATE, + updatedAt: DataTypes.DATE, }, - {}, + { underscored: true }, ); - update_response.associate = () => { + updateResponse.associate = () => { // associations can be defined here }; - return update_response; + return updateResponse; }; diff --git a/src/modules/repository/implementation/sequelize/models/update.js b/src/modules/repository/implementation/sequelize/models/update.js index 422dfab92..e3e07cb87 100644 --- a/src/modules/repository/implementation/sequelize/models/update.js +++ b/src/modules/repository/implementation/sequelize/models/update.js @@ -7,12 +7,12 @@ export default (sequelize, DataTypes) => { primaryKey: true, autoIncrement: true, }, - operation_id: DataTypes.UUID, + operationId: DataTypes.UUID, status: DataTypes.STRING, - created_at: DataTypes.DATE, - updated_at: DataTypes.DATE, + createdAt: DataTypes.DATE, + updatedAt: DataTypes.DATE, }, - {}, + { underscored: true }, ); update.associate = () => { // associations can be defined here diff --git a/src/modules/repository/implementation/sequelize/models/user.js b/src/modules/repository/implementation/sequelize/models/user.js index 5b8d5e4c1..45dfdc297 100644 --- a/src/modules/repository/implementation/sequelize/models/user.js +++ b/src/modules/repository/implementation/sequelize/models/user.js @@ -6,8 +6,8 @@ export default (sequelize, DataTypes) => { type: DataTypes.STRING, unique: true, }, - created_at: DataTypes.DATE, - updated_at: DataTypes.DATE, + createdAt: DataTypes.DATE, + updatedAt: DataTypes.DATE, }, { underscored: true }, ); diff --git a/src/modules/repository/implementation/sequelize/repositories/blockchain-event-repository.js b/src/modules/repository/implementation/sequelize/repositories/blockchain-event-repository.js index 4e9b42c8b..9d21770b4 100644 --- a/src/modules/repository/implementation/sequelize/repositories/blockchain-event-repository.js +++ b/src/modules/repository/implementation/sequelize/repositories/blockchain-event-repository.js @@ -13,7 +13,7 @@ class BlockchainEventRepository { event: event.event, data: event.data, block: event.block, - blockchain_id: event.blockchainId, + blockchainId: event.blockchainId, processed: false, })), { @@ -40,7 +40,7 @@ class BlockchainEventRepository { event, data, block, - blockchain_id: blockchainId, + blockchainId, }, }); return !!dbEvent; diff --git a/src/modules/repository/implementation/sequelize/repositories/blockchain-repository.js b/src/modules/repository/implementation/sequelize/repositories/blockchain-repository.js index a07e463fb..8d16e1802 100644 --- a/src/modules/repository/implementation/sequelize/repositories/blockchain-repository.js +++ b/src/modules/repository/implementation/sequelize/repositories/blockchain-repository.js @@ -6,9 +6,7 @@ class BlockchainRepository { async getLastCheckedBlock(blockchainId, contract) { return this.model.findOne({ - attributes: ['last_checked_block', 'last_checked_timestamp'], - where: { blockchain_id: blockchainId, contract }, - raw: true, + where: { blockchainId, contract }, }); } @@ -22,10 +20,10 @@ class BlockchainRepository { async updateLastCheckedBlock(blockchainId, currentBlock, timestamp, contract) { return this.model.upsert({ - blockchain_id: blockchainId, + blockchainId, contract, - last_checked_block: currentBlock, - last_checked_timestamp: timestamp, + lastCheckedBlock: currentBlock, + lastCheckedTimestamp: timestamp, }); } } diff --git a/src/modules/repository/implementation/sequelize/repositories/command-repository.js b/src/modules/repository/implementation/sequelize/repositories/command-repository.js index 5ad957f25..b272f1b71 100644 --- a/src/modules/repository/implementation/sequelize/repositories/command-repository.js +++ b/src/modules/repository/implementation/sequelize/repositories/command-repository.js @@ -45,7 +45,7 @@ class CommandRepository { await this.model.destroy({ where: { status: { [Sequelize.Op.in]: finalizedStatuses }, - started_at: { [Sequelize.Op.lte]: Date.now() }, + startedAt: { [Sequelize.Op.lte]: Date.now() }, }, }); } diff --git a/src/modules/repository/implementation/sequelize/repositories/event-repository.js b/src/modules/repository/implementation/sequelize/repositories/event-repository.js index 2e342953f..ba2a618b6 100644 --- a/src/modules/repository/implementation/sequelize/repositories/event-repository.js +++ b/src/modules/repository/implementation/sequelize/repositories/event-repository.js @@ -13,7 +13,7 @@ class EventRepository { async createEventRecord(operationId, name, timestamp, value1, value2, value3) { return this.model.create({ - operation_id: operationId, + operationId, name, timestamp, value1, @@ -58,7 +58,7 @@ class EventRepository { return this.model.findAll({ where: { - operation_id: { + operationId: { [Sequelize.Op.in]: operationIds, }, }, diff --git a/src/modules/repository/implementation/sequelize/repositories/operation-id-repository.js b/src/modules/repository/implementation/sequelize/repositories/operation-id-repository.js index cb1664fa2..9d85d2da7 100644 --- a/src/modules/repository/implementation/sequelize/repositories/operation-id-repository.js +++ b/src/modules/repository/implementation/sequelize/repositories/operation-id-repository.js @@ -13,7 +13,7 @@ class OperationIdRepository { async getOperationIdRecord(operationId) { return this.model.findOne({ where: { - operation_id: operationId, + operationId, }, }); } @@ -21,7 +21,7 @@ class OperationIdRepository { async updateOperationIdRecord(data, operationId) { await this.model.update(data, { where: { - operation_id: operationId, + operationId, }, }); } diff --git a/src/modules/repository/implementation/sequelize/repositories/operation-repository.js b/src/modules/repository/implementation/sequelize/repositories/operation-repository.js index 6d5d2a42e..63efad192 100644 --- a/src/modules/repository/implementation/sequelize/repositories/operation-repository.js +++ b/src/modules/repository/implementation/sequelize/repositories/operation-repository.js @@ -6,7 +6,7 @@ class OperationRepository { async createOperationRecord(operation, operationId, status) { return this.models[operation].create({ - operation_id: operationId, + operationId, status, }); } @@ -15,7 +15,7 @@ class OperationRepository { return this.models[operation].findOne({ attributes: ['status'], where: { - operation_id: operationId, + operationId, }, }); } @@ -25,7 +25,7 @@ class OperationRepository { { status }, { where: { - operation_id: operationId, + operationId, }, }, ); diff --git a/src/modules/repository/implementation/sequelize/repositories/operation-response.js b/src/modules/repository/implementation/sequelize/repositories/operation-response.js index b72d9c880..87854a2b2 100644 --- a/src/modules/repository/implementation/sequelize/repositories/operation-response.js +++ b/src/modules/repository/implementation/sequelize/repositories/operation-response.js @@ -12,7 +12,7 @@ class OperationResponseRepository { await this.models[`${operation}_response`].create({ status, message, - operation_id: operationId, + operationId, keyword, }); } @@ -21,7 +21,7 @@ class OperationResponseRepository { return this.models[`${operation}_response`].findAll({ attributes: ['status', 'keyword'], where: { - operation_id: operationId, + operationId, }, }); } diff --git a/src/modules/repository/implementation/sequelize/repositories/service-agreement-repository.js b/src/modules/repository/implementation/sequelize/repositories/service-agreement-repository.js index 275bc8510..abdd54a0c 100644 --- a/src/modules/repository/implementation/sequelize/repositories/service-agreement-repository.js +++ b/src/modules/repository/implementation/sequelize/repositories/service-agreement-repository.js @@ -8,16 +8,16 @@ class ServiceAgreementRepository { async updateServiceAgreementEpochsNumber(agreementId, epochsNumber) { return this.model.update( - { epochs_number: epochsNumber }, + { epochsNumber }, { - where: { agreement_id: agreementId }, + where: { agreementId }, }, ); } async removeServiceAgreements(agreementIds) { return this.model.destroy({ - where: { agreement_id: { [Sequelize.Op.in]: agreementIds } }, + where: { agreementId: { [Sequelize.Op.in]: agreementIds } }, }); } @@ -39,21 +39,21 @@ class ServiceAgreementRepository { lastProofEpoch, ) { return this.model.upsert({ - blockchain_id: blockchainId, - asset_storage_contract_address: contract, - token_id: tokenId, - agreement_id: agreementId, - start_time: startTime, - epochs_number: epochsNumber, - epoch_length: epochLength, - score_function_id: scoreFunctionId, - proof_window_offset_perc: proofWindowOffsetPerc, - hash_function_id: hashFunctionId, + blockchainId, + assetStorageContractAddress: contract, + tokenId, + agreementId, + startTime, + epochsNumber, + epochLength, + scoreFunctionId, + proofWindowOffsetPerc, + hashFunctionId, keyword, - assertion_id: assertionId, - state_index: stateIndex, - last_commit_epoch: lastCommitEpoch, - last_proof_epoch: lastProofEpoch, + assertionId, + stateIndex, + lastCommitEpoch, + lastProofEpoch, }); } @@ -65,10 +65,10 @@ class ServiceAgreementRepository { async updateServiceAgreementLastCommitEpoch(agreementId, lastCommitEpoch) { return this.model.update( - { last_commit_epoch: lastCommitEpoch }, + { lastCommitEpoch }, { where: { - agreement_id: agreementId, + agreementId, }, }, ); @@ -76,10 +76,10 @@ class ServiceAgreementRepository { async updateServiceAgreementLastProofEpoch(agreementId, lastProofEpoch) { return this.model.update( - { last_proof_epoch: lastProofEpoch }, + { lastProofEpoch }, { where: { - agreement_id: agreementId, + agreementId, }, }, ); @@ -88,9 +88,9 @@ class ServiceAgreementRepository { async removeServiceAgreementRecord(blockchainId, contract, tokenId) { await this.model.destroy({ where: { - blockchain_id: blockchainId, - asset_storage_contract_address: contract, - token_id: tokenId, + blockchainId, + assetStorageContractAddress: contract, + tokenId, }, }); } @@ -102,26 +102,26 @@ class ServiceAgreementRepository { return this.model.findAll({ attributes: { include: [ - [Sequelize.literal(currentEpoch), 'current_epoch'], + [Sequelize.literal(currentEpoch), 'currentEpoch'], [ Sequelize.cast( Sequelize.literal(`${commitWindowDurationPerc} - ${currentEpochPerc}`), 'DOUBLE', ), - 'time_left_in_submit_commit_window', + 'timeLeftInSubmitCommitWindow', ], ], }, where: { - blockchain_id: blockchain, + blockchainId: blockchain, [Sequelize.Op.or]: [ { - last_commit_epoch: { + lastCommitEpoch: { [Sequelize.Op.is]: null, }, }, { - last_commit_epoch: { + lastCommitEpoch: { [Sequelize.Op.lt]: Sequelize.literal(currentEpoch), }, }, @@ -129,11 +129,11 @@ class ServiceAgreementRepository { [Sequelize.Op.and]: Sequelize.literal( `${currentEpochPerc} < ${commitWindowDurationPerc}`, ), - epochs_number: { + epochsNumber: { [Sequelize.Op.gt]: Sequelize.literal(currentEpoch), }, }, - order: [[Sequelize.col('time_left_in_submit_commit_window'), 'ASC']], + order: [[Sequelize.col('timeLeftInSubmitCommitWindow'), 'ASC']], limit: 100, raw: true, }); @@ -150,7 +150,7 @@ class ServiceAgreementRepository { return this.model.findAll({ attributes: { include: [ - [Sequelize.literal(currentEpoch), 'current_epoch'], + [Sequelize.literal(currentEpoch), 'currentEpoch'], [ Sequelize.cast( Sequelize.literal( @@ -158,38 +158,38 @@ class ServiceAgreementRepository { ), 'DOUBLE', ), - 'time_left_in_submit_proof_window', + 'timeLeftInSubmitProofWindow', ], ], }, where: { - blockchain_id: blockchain, - last_commit_epoch: { + blockchainId: blockchain, + lastCommitEpoch: { [Sequelize.Op.eq]: Sequelize.literal(currentEpoch), }, [Sequelize.Op.or]: [ { - last_proof_epoch: { + lastProofEpoch: { [Sequelize.Op.is]: null, }, }, { - last_proof_epoch: { + lastProofEpoch: { [Sequelize.Op.lt]: Sequelize.literal(currentEpoch), }, }, ], - proof_window_offset_perc: { + proofWindowOffsetPerc: { [Sequelize.Op.lte]: Sequelize.literal(`${currentEpochPerc}`), [Sequelize.Op.gt]: Sequelize.literal( `${currentEpochPerc} - ${proofWindowDurationPerc}`, ), }, - epochs_number: { + epochsNumber: { [Sequelize.Op.gt]: Sequelize.literal(currentEpoch), }, }, - order: [[Sequelize.col('time_left_in_submit_proof_window'), 'ASC']], + order: [[Sequelize.col('timeLeftInSubmitProofWindow'), 'ASC']], limit: 100, raw: true, }); diff --git a/src/modules/repository/implementation/sequelize/repositories/shard-repository.js b/src/modules/repository/implementation/sequelize/repositories/shard-repository.js index 6c7d9af8e..82fbefd1c 100644 --- a/src/modules/repository/implementation/sequelize/repositories/shard-repository.js +++ b/src/modules/repository/implementation/sequelize/repositories/shard-repository.js @@ -25,20 +25,20 @@ class ShardRepository { ); } - async removeShardingTablePeerRecords(blockchain) { + async removeShardingTablePeerRecords(blockchainId) { return this.model.destroy({ - where: { blockchain_id: blockchain }, + where: { blockchainId }, }); } - async createPeerRecord(peerId, blockchain, ask, stake, lastSeen, sha256) { + async createPeerRecord(peerId, blockchainId, ask, stake, lastSeen, sha256) { return this.model.create( { - peer_id: peerId, - blockchain_id: blockchain, + peerId, + blockchainId, ask, stake, - last_seen: lastSeen, + lastSeen, sha256, }, { @@ -47,18 +47,15 @@ class ShardRepository { ); } - async getAllPeerRecords(blockchain, filterLastSeen) { + async getAllPeerRecords(blockchainId, filterLastSeen) { const query = { where: { - blockchain_id: { - [Sequelize.Op.eq]: blockchain, - }, + blockchainId, }, - raw: true, }; if (filterLastSeen) { - query.where.last_seen = { + query.where.lastSeen = { [Sequelize.Op.gte]: Sequelize.col('last_dialed'), }; } @@ -66,33 +63,28 @@ class ShardRepository { return this.model.findAll(query); } - async getPeerRecord(peerId, blockchain) { + async getPeerRecord(peerId, blockchainId) { return this.model.findOne({ where: { - blockchain_id: { - [Sequelize.Op.eq]: blockchain, - }, - peer_id: { - [Sequelize.Op.eq]: peerId, - }, + blockchainId, + peerId, }, - raw: true, }); } - async getPeersCount(blockchain) { + async getPeersCount(blockchainId) { return this.model.count({ where: { - blockchain_id: blockchain, + blockchainId, }, }); } async getPeersToDial(limit, dialFrequencyMillis) { - return this.model.findAll({ + const result = await this.model.findAll({ attributes: ['peer_id'], where: { - last_dialed: { + lastDialed: { [Sequelize.Op.lt]: new Date(Date.now() - dialFrequencyMillis), }, }, @@ -100,6 +92,7 @@ class ShardRepository { limit, raw: true, }); + return (result ?? []).map((record) => ({ peerId: record.peer_id })); } async updatePeersAsk(peerRecords) { @@ -113,10 +106,10 @@ class ShardRepository { async updatePeerRecordLastDialed(peerId, timestamp) { await this.model.update( { - last_dialed: timestamp, + lastDialed: timestamp, }, { - where: { peer_id: peerId }, + where: { peerId }, }, ); } @@ -124,11 +117,11 @@ class ShardRepository { async updatePeerRecordLastSeenAndLastDialed(peerId, timestamp) { await this.model.update( { - last_dialed: timestamp, - last_seen: timestamp, + lastDialed: timestamp, + lastSeen: timestamp, }, { - where: { peer_id: peerId }, + where: { peerId }, }, ); } @@ -139,7 +132,7 @@ class ShardRepository { async cleanShardingTable(blockchainId) { await this.model.destroy({ - where: blockchainId ? { blockchain_id: blockchainId } : {}, + where: blockchainId ? { blockchainId } : {}, }); } } diff --git a/src/modules/repository/implementation/sequelize/repositories/token-repository.js b/src/modules/repository/implementation/sequelize/repositories/token-repository.js index dc0015f20..6c3e60126 100644 --- a/src/modules/repository/implementation/sequelize/repositories/token-repository.js +++ b/src/modules/repository/implementation/sequelize/repositories/token-repository.js @@ -9,8 +9,8 @@ class TokenRepository { async saveToken(tokenId, userId, tokenName, expiresAt) { return this.model.create({ id: tokenId, - user_id: userId, - expires_at: expiresAt, + userId, + expiresAt, name: tokenName, }); } diff --git a/src/service/blockchain-event-listener-service.js b/src/service/blockchain-event-listener-service.js index 0cb633eb2..52fe36db7 100644 --- a/src/service/blockchain-event-listener-service.js +++ b/src/service/blockchain-event-listener-service.js @@ -123,11 +123,12 @@ class BlockchainEventListenerService { blockchainId, contractName, ); + const events = await this.blockchainModuleManager.getAllPastEvents( blockchainId, contractName, - lastCheckedBlockObject?.last_checked_block ?? 0, - lastCheckedBlockObject?.last_checked_timestamp ?? 0, + lastCheckedBlockObject?.lastCheckedBlock ?? 0, + lastCheckedBlockObject?.lastCheckedTimestamp ?? 0, currentBlock, ); @@ -200,7 +201,7 @@ class BlockchainEventListenerService { for (const event of blockEvents) { const { contractName, newContractAddress } = JSON.parse(event.data); this.blockchainModuleManager.initializeContract( - event.blockchain_id, + event.blockchainId, contractName, newContractAddress, ); @@ -212,13 +213,13 @@ class BlockchainEventListenerService { blockEvents.map(async (event) => { const { contractName, newContractAddress } = JSON.parse(event.data); this.blockchainModuleManager.initializeContract( - event.blockchain_id, + event.blockchainId, contractName, newContractAddress, ); if (contractName === CONTRACTS.SHARDING_TABLE_CONTRACT) { - await this.repositoryModuleManager.cleanShardingTable(event.blockchain_id); + await this.repositoryModuleManager.cleanShardingTable(event.blockchainId); } }), ); @@ -228,7 +229,7 @@ class BlockchainEventListenerService { for (const event of blockEvents) { const { newContractAddress } = JSON.parse(event.data); this.blockchainModuleManager.initializeAssetStorageContract( - event.blockchain_id, + event.blockchainId, newContractAddress, ); } @@ -238,7 +239,7 @@ class BlockchainEventListenerService { for (const event of blockEvents) { const { newContractAddress } = JSON.parse(event.data); this.blockchainModuleManager.initializeAssetStorageContract( - event.blockchain_id, + event.blockchainId, newContractAddress, ); } @@ -250,7 +251,7 @@ class BlockchainEventListenerService { const eventData = JSON.parse(event.data); const nodeId = this.blockchainModuleManager.convertHexToAscii( - event.blockchain_id, + event.blockchainId, eventData.nodeId, ); @@ -262,17 +263,17 @@ class BlockchainEventListenerService { this.logger.trace(`Adding peer id: ${nodeId} to sharding table.`); return { - peer_id: nodeId, - blockchain_id: event.blockchain_id, + peerId: nodeId, + blockchainId: event.blockchainId, ask: this.blockchainModuleManager.convertFromWei( - event.blockchain_id, + event.blockchainId, eventData.ask, ), stake: this.blockchainModuleManager.convertFromWei( - event.blockchain_id, + event.blockchainId, eventData.stake, ), - last_seen: new Date(0), + lastSeen: new Date(0), sha256: nodeIdSha256, }; }), @@ -286,14 +287,14 @@ class BlockchainEventListenerService { const eventData = JSON.parse(event.data); const nodeId = this.blockchainModuleManager.convertHexToAscii( - event.blockchain_id, + event.blockchainId, eventData.nodeId, ); this.logger.trace(`Removing peer id: ${nodeId} from sharding table.`); return { - peer_id: nodeId, - blockchain_id: event.blockchain_id, + peerId: nodeId, + blockchainId: event.blockchainId, }; }), ); @@ -307,17 +308,17 @@ class BlockchainEventListenerService { const eventData = JSON.parse(event.data); const nodeId = this.blockchainModuleManager.convertHexToAscii( - event.blockchain_id, + event.blockchainId, eventData.nodeId, ); this.logger.trace(`Updating stake value for peer id: ${nodeId} in sharding table.`); return { - peer_id: nodeId, - blockchain_id: event.blockchain_id, + peerId: nodeId, + blockchainId: event.blockchainId, stake: this.blockchainModuleManager.convertFromWei( - event.blockchain_id, + event.blockchainId, eventData.newStake, ), }; @@ -337,17 +338,17 @@ class BlockchainEventListenerService { const eventData = JSON.parse(event.data); const nodeId = this.blockchainModuleManager.convertHexToAscii( - event.blockchain_id, + event.blockchainId, eventData.nodeId, ); this.logger.trace(`Updating ask value for peer id: ${nodeId} in sharding table.`); return { - peer_id: nodeId, - blockchain_id: event.blockchain_id, + peerId: nodeId, + blockchainId: event.blockchainId, ask: this.blockchainModuleManager.convertFromWei( - event.blockchain_id, + event.blockchainId, eventData.ask, ), }; @@ -363,7 +364,7 @@ class BlockchainEventListenerService { const { agreementId } = JSON.parse(event.data); const { epochsNumber } = await this.blockchainModuleManager.getAgreementData( - event.blockchain_id, + event.blockchainId, agreementId, ); @@ -387,7 +388,7 @@ class BlockchainEventListenerService { const eventData = JSON.parse(event.data); const { tokenId, keyword, state, stateIndex } = eventData; - const blockchain = event.blockchain_id; + const blockchain = event.blockchainId; const contract = eventData.assetContract; this.logger.trace( `Handling event: ${event.event} for asset with ual: ${this.ualService.deriveUAL( diff --git a/src/service/operation-id-service.js b/src/service/operation-id-service.js index fd6c744cb..eb727cbe7 100644 --- a/src/service/operation-id-service.js +++ b/src/service/operation-id-service.js @@ -15,7 +15,7 @@ class OperationIdService { const operationIdObject = await this.repositoryModuleManager.createOperationIdRecord({ status, }); - const operationId = operationIdObject.operation_id; + const { operationId } = operationIdObject; this.emitChangeEvent(status, operationId); this.logger.debug(`Generated operation id for request ${operationId}`); return operationId; diff --git a/src/service/sharding-table-service.js b/src/service/sharding-table-service.js index 7f92f6aff..2a09e598e 100644 --- a/src/service/sharding-table-service.js +++ b/src/service/sharding-table-service.js @@ -44,8 +44,8 @@ class ShardingTableService { ); if ( - lastCheckedBlock?.last_checked_timestamp && - Date.now() - lastCheckedBlock.last_checked_timestamp < + lastCheckedBlock?.lastCheckedTimestamp && + Date.now() - lastCheckedBlock.lastCheckedTimestamp < DEFAULT_BLOCKCHAIN_EVENT_SYNC_PERIOD_IN_MILLS ) { return; @@ -95,8 +95,8 @@ class ShardingTableService { ); return { - peer_id: nodeId, - blockchain_id: blockchainId, + peerId: nodeId, + blockchainId, ask: this.blockchainModuleManager.convertFromWei( blockchainId, peer.ask, diff --git a/test/modules/repository/repository.js b/test/modules/repository/repository.js index 3f9a78d7d..05955685c 100644 --- a/test/modules/repository/repository.js +++ b/test/modules/repository/repository.js @@ -11,49 +11,48 @@ const config = JSON.parse(await readFile('./test/modules/repository/config.json' const blockchain = 'hardhat'; const createAgreement = ({ - blockchain_id = blockchain, - asset_storage_contract_address = '0xB0D4afd8879eD9F52b28595d31B441D079B2Ca07', - token_id, - agreement_id = null, - start_time, - epochs_number = 2, - epoch_length = 100, - score_function_id = 1, - proof_window_offset_perc = 66, - hash_function_id = 1, + blockchainId = blockchain, + assetStorageContractAddress = '0xB0D4afd8879eD9F52b28595d31B441D079B2Ca07', + tokenId, + id = null, + startTime, + epochsNumber = 2, + epochLength = 100, + scoreFunctionId = 1, + proofWindowOffsetPerc = 66, + hashFunctionId = 1, keyword = '0xB0D4afd8879eD9F52b28595d31B441D079B2Ca0768e44dc71bf509adfccbea9df949f253afa56796a3a926203f90a1e4914247d3', - assertion_id = '0x68e44dc71bf509adfccbea9df949f253afa56796a3a926203f90a1e4914247d3', - state_index = 1, - last_commit_epoch = null, - last_proof_epoch = null, + assertionId = '0x68e44dc71bf509adfccbea9df949f253afa56796a3a926203f90a1e4914247d3', + stateIndex = 1, + lastCommitEpoch = null, + lastProofEpoch = null, }) => { const agreementId = - agreement_id ?? + id ?? utils.sha256( utils.toUtf8Bytes( utils.solidityPack( ['address', 'uint256', 'bytes'], - [asset_storage_contract_address, token_id, keyword], + [assetStorageContractAddress, tokenId, keyword], ), ), ); - return { - blockchain_id, - asset_storage_contract_address, - token_id, - agreement_id: agreementId, - start_time, - epochs_number, - epoch_length, - score_function_id, - proof_window_offset_perc, - hash_function_id, + blockchainId, + assetStorageContractAddress, + tokenId, + agreementId, + startTime, + epochsNumber, + epochLength, + scoreFunctionId, + proofWindowOffsetPerc, + hashFunctionId, keyword, - assertion_id, - state_index, - last_commit_epoch, - last_proof_epoch, + assertionId, + stateIndex, + lastCommitEpoch, + lastProofEpoch, }; }; @@ -93,100 +92,98 @@ describe('Repository module', () => { }); describe('Insert and update service agreement', () => { const agreement = { - blockchain_id: blockchain, - asset_storage_contract_address: '0xB0D4afd8879eD9F52b28595d31B441D079B2Ca07', - token_id: 0, - agreement_id: '0x44cf660357e2d7462c25fd8e50b68abe332d7a70b07a76e92f628846ea585881', - start_time: 1683032289, - epochs_number: 2, - epoch_length: 360, - score_function_id: 1, - proof_window_offset_perc: 66, - hash_function_id: 1, + blockchainId: blockchain, + assetStorageContractAddress: '0xB0D4afd8879eD9F52b28595d31B441D079B2Ca07', + tokenId: 0, + agreementId: '0x44cf660357e2d7462c25fd8e50b68abe332d7a70b07a76e92f628846ea585881', + startTime: 1683032289, + epochsNumber: 2, + epochLength: 360, + scoreFunctionId: 1, + proofWindowOffsetPerc: 66, + hashFunctionId: 1, keyword: '0xB0D4afd8879eD9F52b28595d31B441D079B2Ca0768e44dc71bf509adfccbea9df949f253afa56796a3a926203f90a1e4914247d3', - assertion_id: '0x68e44dc71bf509adfccbea9df949f253afa56796a3a926203f90a1e4914247d3', - state_index: 1, + assertionId: '0x68e44dc71bf509adfccbea9df949f253afa56796a3a926203f90a1e4914247d3', + stateIndex: 1, }; it('inserts service agreement', async () => { const inserted = await repositoryModuleManager.updateServiceAgreementRecord( - agreement.blockchain_id, - agreement.asset_storage_contract_address, - agreement.token_id, - agreement.agreement_id, - agreement.start_time, - agreement.epochs_number, - agreement.epoch_length, - agreement.score_function_id, - agreement.proof_window_offset_perc, - agreement.hash_function_id, + agreement.blockchainId, + agreement.assetStorageContractAddress, + agreement.tokenId, + agreement.agreementId, + agreement.startTime, + agreement.epochsNumber, + agreement.epochLength, + agreement.scoreFunctionId, + agreement.proofWindowOffsetPerc, + agreement.hashFunctionId, agreement.keyword, - agreement.assertion_id, - agreement.state_index, - agreement.last_commit_epoch, - agreement.last_proof_epoch, + agreement.assertionId, + agreement.stateIndex, + agreement.lastCommitEpoch, + agreement.lastProofEpoch, ); const row = inserted[0]?.dataValues; assert(expect(row).to.exist); - expect(row.blockchain_id).to.equal(agreement.blockchain_id); - expect(row.asset_storage_contract_address).to.equal( - agreement.asset_storage_contract_address, - ); - expect(row.token_id).to.equal(agreement.token_id); - expect(row.agreement_id).to.equal(agreement.agreement_id); - expect(row.start_time).to.equal(agreement.start_time); - expect(row.epochs_number).to.equal(agreement.epochs_number); - expect(row.epoch_length).to.equal(agreement.epoch_length); - expect(row.score_function_id).to.equal(agreement.score_function_id); - expect(row.proof_window_offset_perc).to.equal(agreement.proof_window_offset_perc); - expect(row.hash_function_id).to.equal(agreement.hash_function_id); + expect(row.blockchainId).to.equal(agreement.blockchainId); + expect(row.assetStorageContractAddress).to.equal(agreement.assetStorageContractAddress); + expect(row.tokenId).to.equal(agreement.tokenId); + expect(row.agreementId).to.equal(agreement.agreementId); + expect(row.startTime).to.equal(agreement.startTime); + expect(row.epochsNumber).to.equal(agreement.epochsNumber); + expect(row.epochLength).to.equal(agreement.epochLength); + expect(row.scoreFunctionId).to.equal(agreement.scoreFunctionId); + expect(row.proofWindowOffsetPerc).to.equal(agreement.proofWindowOffsetPerc); + expect(row.hashFunctionId).to.equal(agreement.hashFunctionId); expect(row.keyword).to.equal(agreement.keyword); - expect(row.assertion_id).to.equal(agreement.assertion_id); - expect(row.state_index).to.equal(agreement.state_index); - assert(expect(row.last_commit_epoch).to.not.exist); - assert(expect(row.last_proof_epoch).to.not.exist); + expect(row.assertionId).to.equal(agreement.assertionId); + expect(row.stateIndex).to.equal(agreement.stateIndex); + assert(expect(row.lastCommitEpoch).to.not.exist); + assert(expect(row.lastProofEpoch).to.not.exist); }); }); describe('Eligible service agreements', () => { const agreements = [ - createAgreement({ token_id: 0, start_time: 0 }), + createAgreement({ tokenId: 0, startTime: 0 }), createAgreement({ - token_id: 1, - start_time: 15, - last_commit_epoch: 0, + tokenId: 1, + startTime: 15, + lastCommitEpoch: 0, }), - createAgreement({ token_id: 2, start_time: 25 }), + createAgreement({ tokenId: 2, startTime: 25 }), createAgreement({ - token_id: 3, - start_time: 25, - last_commit_epoch: 0, - last_proof_epoch: 0, + tokenId: 3, + startTime: 25, + lastCommitEpoch: 0, + lastProofEpoch: 0, }), - createAgreement({ token_id: 4, start_time: 49 }), + createAgreement({ tokenId: 4, startTime: 49 }), ]; beforeEach(async () => { await Promise.all( agreements.map((agreement) => repositoryModuleManager.updateServiceAgreementRecord( - agreement.blockchain_id, - agreement.asset_storage_contract_address, - agreement.token_id, - agreement.agreement_id, - agreement.start_time, - agreement.epochs_number, - agreement.epoch_length, - agreement.score_function_id, - agreement.proof_window_offset_perc, - agreement.hash_function_id, + agreement.blockchainId, + agreement.assetStorageContractAddress, + agreement.tokenId, + agreement.agreementId, + agreement.startTime, + agreement.epochsNumber, + agreement.epochLength, + agreement.scoreFunctionId, + agreement.proofWindowOffsetPerc, + agreement.hashFunctionId, agreement.keyword, - agreement.assertion_id, - agreement.state_index, - agreement.last_commit_epoch, - agreement.last_proof_epoch, + agreement.assertionId, + agreement.stateIndex, + agreement.lastCommitEpoch, + agreement.lastProofEpoch, ), ), ); @@ -210,8 +207,8 @@ describe('Repository module', () => { // ensure order is correct for (let i = 0; i < eligibleAgreements.length; i += 1) { assert.strictEqual( - eligibleAgreements[i].time_left_in_submit_commit_window, - expectedAgreements[i].time_left_in_submit_commit_window, + eligibleAgreements[i].timeLeftInSubmitCommitWindow, + expectedAgreements[i].timeLeftInSubmitCommitWindow, ); } }; @@ -219,14 +216,14 @@ describe('Repository module', () => { it( 'returns two eligible service agreements at timestamp 49', testEligibleAgreementsForSubmitCommit(49, 25, [ - { ...agreements[2], current_epoch: 0, time_left_in_submit_commit_window: 1 }, - { ...agreements[4], current_epoch: 0, time_left_in_submit_commit_window: 25 }, + { ...agreements[2], currentEpoch: 0, timeLeftInSubmitCommitWindow: 1 }, + { ...agreements[4], currentEpoch: 0, timeLeftInSubmitCommitWindow: 25 }, ]), ); it( 'returns one eligible service agreement at timestamp 51', testEligibleAgreementsForSubmitCommit(51, 25, [ - { ...agreements[4], current_epoch: 0, time_left_in_submit_commit_window: 23 }, + { ...agreements[4], currentEpoch: 0, timeLeftInSubmitCommitWindow: 23 }, ]), ); it( @@ -240,44 +237,44 @@ describe('Repository module', () => { it( 'returns one eligible service agreements at timestamp 100', testEligibleAgreementsForSubmitCommit(100, 25, [ - { ...agreements[0], current_epoch: 1, time_left_in_submit_commit_window: 25 }, + { ...agreements[0], currentEpoch: 1, timeLeftInSubmitCommitWindow: 25 }, ]), ); it( 'returns two eligible service agreements at timestamp 124', testEligibleAgreementsForSubmitCommit(124, 25, [ - { ...agreements[0], current_epoch: 1, time_left_in_submit_commit_window: 1 }, - { ...agreements[1], current_epoch: 1, time_left_in_submit_commit_window: 16 }, + { ...agreements[0], currentEpoch: 1, timeLeftInSubmitCommitWindow: 1 }, + { ...agreements[1], currentEpoch: 1, timeLeftInSubmitCommitWindow: 16 }, ]), ); it( 'returns three eligible service agreements at timestamp 125', testEligibleAgreementsForSubmitCommit(125, 25, [ - { ...agreements[1], current_epoch: 1, time_left_in_submit_commit_window: 15 }, - { ...agreements[2], current_epoch: 1, time_left_in_submit_commit_window: 25 }, - { ...agreements[3], current_epoch: 1, time_left_in_submit_commit_window: 25 }, + { ...agreements[1], currentEpoch: 1, timeLeftInSubmitCommitWindow: 15 }, + { ...agreements[2], currentEpoch: 1, timeLeftInSubmitCommitWindow: 25 }, + { ...agreements[3], currentEpoch: 1, timeLeftInSubmitCommitWindow: 25 }, ]), ); it( 'returns three eligible service agreements at timestamp 126', testEligibleAgreementsForSubmitCommit(126, 25, [ - { ...agreements[1], current_epoch: 1, time_left_in_submit_commit_window: 14 }, - { ...agreements[2], current_epoch: 1, time_left_in_submit_commit_window: 24 }, - { ...agreements[3], current_epoch: 1, time_left_in_submit_commit_window: 24 }, + { ...agreements[1], currentEpoch: 1, timeLeftInSubmitCommitWindow: 14 }, + { ...agreements[2], currentEpoch: 1, timeLeftInSubmitCommitWindow: 24 }, + { ...agreements[3], currentEpoch: 1, timeLeftInSubmitCommitWindow: 24 }, ]), ); it( 'returns three eligible service agreements at timestamp 149', testEligibleAgreementsForSubmitCommit(149, 25, [ - { ...agreements[2], current_epoch: 1, time_left_in_submit_commit_window: 1 }, - { ...agreements[3], current_epoch: 1, time_left_in_submit_commit_window: 1 }, - { ...agreements[4], current_epoch: 1, time_left_in_submit_commit_window: 25 }, + { ...agreements[2], currentEpoch: 1, timeLeftInSubmitCommitWindow: 1 }, + { ...agreements[3], currentEpoch: 1, timeLeftInSubmitCommitWindow: 1 }, + { ...agreements[4], currentEpoch: 1, timeLeftInSubmitCommitWindow: 25 }, ]), ); it( 'returns one eligible service agreements at timestamp 151', testEligibleAgreementsForSubmitCommit(151, 25, [ - { ...agreements[4], current_epoch: 1, time_left_in_submit_commit_window: 23 }, + { ...agreements[4], currentEpoch: 1, timeLeftInSubmitCommitWindow: 23 }, ]), ); it( @@ -308,8 +305,8 @@ describe('Repository module', () => { // ensure order is correct for (let i = 0; i < eligibleAgreements.length; i += 1) { assert.strictEqual( - eligibleAgreements[i].time_left_in_submit_proof_window, - expectedAgreements[i].time_left_in_submit_proof_window, + eligibleAgreements[i].timeLeftInSubmitProofWindow, + expectedAgreements[i].timeLeftInSubmitProofWindow, ); } }; @@ -329,19 +326,19 @@ describe('Repository module', () => { it( 'returns one eligible service agreements at timestamp 81', testEligibleAgreementsForSubmitProof(81, 33, [ - { ...agreements[1], current_epoch: 0, time_left_in_submit_proof_window: 33 }, + { ...agreements[1], currentEpoch: 0, timeLeftInSubmitProofWindow: 33 }, ]), ); it( 'returns one eligible service agreements at timestamp 92', testEligibleAgreementsForSubmitProof(92, 33, [ - { ...agreements[1], current_epoch: 0, time_left_in_submit_proof_window: 22 }, + { ...agreements[1], currentEpoch: 0, timeLeftInSubmitProofWindow: 22 }, ]), ); it( 'returns one eligible service agreements at timestamp 113', testEligibleAgreementsForSubmitProof(113, 33, [ - { ...agreements[1], current_epoch: 0, time_left_in_submit_proof_window: 1 }, + { ...agreements[1], currentEpoch: 0, timeLeftInSubmitProofWindow: 1 }, ]), ); it( @@ -376,10 +373,10 @@ describe('Repository module', () => { for (let tokenId = 0; tokenId < numAgreements; tokenId += 1) { agreements.push( createAgreement({ - token_id: tokenId, - start_time: Math.floor(Math.random() * 101), - last_commit_epoch: [null, 0][Math.floor(Math.random() * 3)], - last_proof_epoch: [null, 0][Math.floor(Math.random() * 3)], + tokenId, + startTime: Math.floor(Math.random() * 101), + lastCommitEpoch: [null, 0][Math.floor(Math.random() * 3)], + lastProofEpoch: [null, 0][Math.floor(Math.random() * 3)], }), ); diff --git a/test/unit/mock/repository-module-manager-mock.js b/test/unit/mock/repository-module-manager-mock.js index cb9e85e37..ff6aa9316 100644 --- a/test/unit/mock/repository-module-manager-mock.js +++ b/test/unit/mock/repository-module-manager-mock.js @@ -2,105 +2,105 @@ class RepositoryModuleManagerMock { responseStatuses = [ { id: 1, - operation_id: 'f6354c2c-d460-11ed-afa1-0242ac120002', + operationId: 'f6354c2c-d460-11ed-afa1-0242ac120002', keyword: 'origintrail', status: 'COMPLETED', message: 'message', - created_at: '1970-01-01 00:00:00', - updated_at: '1970-01-01 00:00:00', + createdAt: '1970-01-01 00:00:00', + updatedAt: '1970-01-01 00:00:00', }, ]; getAllPeerRecords() { return [ { - peer_id: 'QmcJY13uLyt2VQ6QiVNcYiWaxdfaHWHj3T7G472uaHPBf7', - blockchain_id: 'ganache', + peerId: 'QmcJY13uLyt2VQ6QiVNcYiWaxdfaHWHj3T7G472uaHPBf7', + blockchainId: 'ganache', ask: '0.2824612246520951', stake: '50000.0', - last_seen: '1970-01-01 00:00:00', - last_dialed: '1970-01-01 00:00:00', + lastSeen: '1970-01-01 00:00:00', + lastDialed: '1970-01-01 00:00:00', sha256: '0x6e08776479a010d563855dbc371a66f692d3edcbcf2b02c30f9879ebe02244e8', }, { - peer_id: 'Qmcxo88zf5zEvyBLYTrtfG8nGJQW6zHpf58b5MUcjoYVqL', - blockchain_id: 'ganache', + peerId: 'Qmcxo88zf5zEvyBLYTrtfG8nGJQW6zHpf58b5MUcjoYVqL', + blockchainId: 'ganache', ask: '0.11680988694381877', stake: '50000.0', - last_seen: '1970-01-01 00:00:00', - last_dialed: '1970-01-01 00:00:00', + lastSeen: '1970-01-01 00:00:00', + lastDialed: '1970-01-01 00:00:00', sha256: '0x113d3da32b0e0b7031d188736792bbea0baf7911acb905511ac7dda2be9a6f55', }, { - peer_id: 'QmQeNwBzgeMQxquQEDXvBHqXBHNBEvKHtyHURg4QvnoLrD', - blockchain_id: 'ganache', + peerId: 'QmQeNwBzgeMQxquQEDXvBHqXBHNBEvKHtyHURg4QvnoLrD', + blockchainId: 'ganache', ask: '0.25255488168658036', stake: '50000.0', - last_seen: '1970-01-01 00:00:00', - last_dialed: '1970-01-01 00:00:00', + lastSeen: '1970-01-01 00:00:00', + lastDialed: '1970-01-01 00:00:00', sha256: '0xba14ac66ab5be40bf458bad9b4e9f10a9d06375b233e91a6ce3c2d4cbf9deea5', }, { - peer_id: 'QmU4ty8X8L4Xk6cbDCoyJUhgeBNLDo3HprTGEhNd9CtiT7', - blockchain_id: 'ganache', + peerId: 'QmU4ty8X8L4Xk6cbDCoyJUhgeBNLDo3HprTGEhNd9CtiT7', + blockchainId: 'ganache', ask: '0.25263875217271087', stake: '50000.0', - last_seen: '1970-01-01 00:00:00', - last_dialed: '1970-01-01 00:00:00', + lastSeen: '1970-01-01 00:00:00', + lastDialed: '1970-01-01 00:00:00', sha256: '0x5b3fdb88b3270a99cc89d28e0a4504d28789e5f8ca53080aa7608db48546d56b', }, { - peer_id: 'QmWmgmMCQQ1awraTeQqwsbWgqtR3ZMuX7NhbHyiftuAspb', - blockchain_id: 'ganache', + peerId: 'QmWmgmMCQQ1awraTeQqwsbWgqtR3ZMuX7NhbHyiftuAspb', + blockchainId: 'ganache', ask: '0.2429885059428509', stake: '50000.0', - last_seen: '1970-01-01 00:00:00', - last_dialed: '1970-01-01 00:00:00', + lastSeen: '1970-01-01 00:00:00', + lastDialed: '1970-01-01 00:00:00', sha256: '0x820a8e38cb792b89c8b69eb9c192faf3def6175c97c4c0f17708161bcb9c5028', }, { - peer_id: 'QmWyf3dtqJnhuCpzEDTNmNFYc5tjxTrXhGcUUmGHdg2gtj', - blockchain_id: 'ganache', + peerId: 'QmWyf3dtqJnhuCpzEDTNmNFYc5tjxTrXhGcUUmGHdg2gtj', + blockchainId: 'ganache', ask: '0.210617584797714', stake: '50000.0', - last_seen: '1970-01-01 00:00:00', - last_dialed: '1970-01-01 00:00:00', + lastSeen: '1970-01-01 00:00:00', + lastDialed: '1970-01-01 00:00:00', sha256: '0xf764186e9b675f3fd00af72026cf075d05ce8fc951ba089351d645b363acd3d3', }, { - peer_id: 'QmXgeHgBVbd7iyTp8PapUAyeKciqbsXTEvsakCjW7wZRqT', - blockchain_id: 'ganache', + peerId: 'QmXgeHgBVbd7iyTp8PapUAyeKciqbsXTEvsakCjW7wZRqT', + blockchainId: 'ganache', ask: '0.2290449496761527', stake: '50000.0', - last_seen: '1970-01-01 00:00:00', - last_dialed: '1970-01-01 00:00:00', + lastSeen: '1970-01-01 00:00:00', + lastDialed: '1970-01-01 00:00:00', sha256: '0xaaeed7b766483aef7cf2d07325f336b3e703e2b7573e540ca8c6e2aab34265c3', }, { - peer_id: 'QmYys42KLmGEE9hEmJCVCe3SR3G9zf4epoAwDUK7pVUP6S', - blockchain_id: 'ganache', + peerId: 'QmYys42KLmGEE9hEmJCVCe3SR3G9zf4epoAwDUK7pVUP6S', + blockchainId: 'ganache', ask: '0.1637075464317365', stake: '50000.0', - last_seen: '1970-01-01 00:00:00', - last_dialed: '1970-01-01 00:00:00', + lastSeen: '1970-01-01 00:00:00', + lastDialed: '1970-01-01 00:00:00', sha256: '0xc3bb7b5433ebe62ff9e98c6d439223d07d44e16e7d5e210e727823f87c0ef24b', }, { - peer_id: 'QmZi2nDhZJfa1Z5iXjvxQ1BigpR8TdTQ3gWQDGecn34e9x', - blockchain_id: 'ganache', + peerId: 'QmZi2nDhZJfa1Z5iXjvxQ1BigpR8TdTQ3gWQDGecn34e9x', + blockchainId: 'ganache', ask: '0.10242295311162795', stake: '50000.0', - last_seen: '1970-01-01 00:00:00', - last_dialed: '1970-01-01 00:00:00', + lastSeen: '1970-01-01 00:00:00', + lastDialed: '1970-01-01 00:00:00', sha256: '0x510ca60cdd7b33bf8d978576981ae7f9caaf5f133ddd40693d8ce007614c0a09', }, { - peer_id: 'QmZueq5jip24v5dbCSBGt8v16hPjUN1CXRb3zGaxH1jfHM', - blockchain_id: 'ganache', + peerId: 'QmZueq5jip24v5dbCSBGt8v16hPjUN1CXRb3zGaxH1jfHM', + blockchainId: 'ganache', ask: '0.23374911902136858', stake: '50000.0', - last_seen: '1970-01-01 00:00:00', - last_dialed: '1970-01-01 00:00:00', + lastSeen: '1970-01-01 00:00:00', + lastDialed: '1970-01-01 00:00:00', sha256: '0x7b4f717bd647104a72c7f1fce4600366982f36ebb1cef41540a5541c8e8ca1dd', }, ]; @@ -111,21 +111,21 @@ class RepositoryModuleManagerMock { } async getOperationResponsesStatuses(operation, operationId) { - return this.responseStatuses.filter((rs) => rs.operation_id === operationId); + return this.responseStatuses.filter((rs) => rs.operationId === operationId); } async updateOperationIdRecord(data, operationId) { this.responseStatuses = this.responseStatuses.map((rs) => - rs.operation_id === operationId - ? { ...rs, status: data.status, updated_at: data.timestamp } + rs.operationId === operationId + ? { ...rs, status: data.status, updatedAt: data.timestamp } : rs, ); } async updateOperationStatus(operation, operationId, status) { this.responseStatuses = this.responseStatuses.map((rs) => - rs.operation_id === operationId - ? { ...rs, status, updated_at: new Date().toISOString() } + rs.operationId === operationId + ? { ...rs, status, updatedAt: new Date().toISOString() } : rs, ); } @@ -136,10 +136,10 @@ class RepositoryModuleManagerMock { { id: this.responseStatuses[this.responseStatuses.length - 1].id + 1, status, - operation_id: operationId, + operationId, keyword, - created_at: new Date().toISOString(), - updated_at: new Date().toISOString(), + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), }, ]; } From 62c10652828e9c53ab1b791fb4097d3756548cea Mon Sep 17 00:00:00 2001 From: zeroxbt <89495162+zeroxbt@users.noreply.github.com> Date: Fri, 19 May 2023 14:32:53 +0200 Subject: [PATCH 11/16] limit number of txs based on current queue length (#2537) --- src/commands/protocols/common/epoch-check-command.js | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/commands/protocols/common/epoch-check-command.js b/src/commands/protocols/common/epoch-check-command.js index 552120120..d6bfe4a10 100644 --- a/src/commands/protocols/common/epoch-check-command.js +++ b/src/commands/protocols/common/epoch-check-command.js @@ -20,12 +20,18 @@ class EpochCheckCommand extends Command { await this.blockchainModuleManager.getCommitWindowDurationPerc(blockchain); const proofWindowDurationPerc = await this.blockchainModuleManager.getProofWindowDurationPerc(blockchain); - const totalTransactions = await this.calculateTotalTransactions( + let totalTransactions = await this.calculateTotalTransactions( blockchain, commitWindowDurationPerc, proofWindowDurationPerc, command.period, ); + + const transactionQueueLength = + this.blockchainModuleManager.getTransactionQueueLength(blockchain); + if (transactionQueueLength >= totalTransactions) return; + totalTransactions -= transactionQueueLength; + await Promise.all([ this.scheduleSubmitCommitCommands( blockchain, From e4614c44f48a019b1fdc5f0d6169283d771bceec Mon Sep 17 00:00:00 2001 From: zeroxbt <89495162+zeroxbt@users.noreply.github.com> Date: Mon, 22 May 2023 10:19:54 +0200 Subject: [PATCH 12/16] wait for txs to complete in submit commit and proof commands (#2538) * limit number of txs based on current queue length * wait for txs to complete in submit commit and proof commands * ensure commit and proofs dont block command queue * add check for proof already submitted * bump version --- package-lock.json | 4 +- package.json | 2 +- src/commands/command-executor.js | 11 +- .../protocols/common/epoch-check-command.js | 20 ++- .../protocols/common/submit-commit-command.js | 94 +++++--------- .../protocols/common/submit-proofs-command.js | 119 +++++++----------- .../receiver/submit-update-commit-command.js | 2 +- .../v1-0-0-handle-update-request-command.js | 2 +- src/constants/constants.js | 6 + .../blockchain/blockchain-module-manager.js | 6 +- .../blockchain/implementation/web3-service.js | 15 +-- 11 files changed, 116 insertions(+), 165 deletions(-) diff --git a/package-lock.json b/package-lock.json index 7d1bc7066..0a55642e2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "origintrail_node", - "version": "6.0.9", + "version": "6.0.9+hotfix.2", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "origintrail_node", - "version": "6.0.9", + "version": "6.0.9+hotfix.2", "license": "ISC", "dependencies": { "@comunica/query-sparql": "^2.4.3", diff --git a/package.json b/package.json index 9a9ee6a17..6c27375d2 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "origintrail_node", - "version": "6.0.9", + "version": "6.0.9+hotfix.2", "description": "OTNode V6", "main": "index.js", "type": "module", diff --git a/src/commands/command-executor.js b/src/commands/command-executor.js index 963cef3fd..e44734f71 100644 --- a/src/commands/command-executor.js +++ b/src/commands/command-executor.js @@ -6,14 +6,9 @@ import { DEFAULT_COMMAND_REPEAT_INTERVAL_IN_MILLS, COMMAND_STATUS, DEFAULT_COMMAND_DELAY_IN_MILLS, + COMMAND_QUEUE_PARALLELISM, } from '../constants/constants.js'; -/** - * How many commands will run in parallel - * @type {number} - */ -const QUEUE_PARALLELISM = 100; - /** * Queues and processes commands */ @@ -26,8 +21,6 @@ class CommandExecutor { this.started = false; this.repositoryModuleManager = ctx.repositoryModuleManager; - - this.parallelism = QUEUE_PARALLELISM; this.verboseLoggingEnabled = this.config.commandExecutorVerboseLoggingEnabled; this.queue = async.queue((command, callback = () => {}) => { @@ -41,7 +34,7 @@ class CommandExecutor { ); process.exit(1); }); - }, this.parallelism); + }, COMMAND_QUEUE_PARALLELISM); } /** diff --git a/src/commands/protocols/common/epoch-check-command.js b/src/commands/protocols/common/epoch-check-command.js index d6bfe4a10..1ab33426f 100644 --- a/src/commands/protocols/common/epoch-check-command.js +++ b/src/commands/protocols/common/epoch-check-command.js @@ -1,6 +1,11 @@ /* eslint-disable no-await-in-loop */ +import { v4 as uuidv4 } from 'uuid'; import Command from '../../command.js'; -import { COMMAND_RETRIES, TRANSACTION_CONFIRMATIONS } from '../../../constants/constants.js'; +import { + COMMAND_QUEUE_PARALLELISM, + COMMAND_RETRIES, + TRANSACTION_CONFIRMATIONS, +} from '../../../constants/constants.js'; class EpochCheckCommand extends Command { constructor(ctx) { @@ -27,6 +32,11 @@ class EpochCheckCommand extends Command { command.period, ); + // We don't expect to have this many transactions in one epoch check window. + // This is just to make sure we don't schedule too many commands and block the queue + // TODO: find general solution for all commands scheduling blockchain transactions + totalTransactions = Math.min(totalTransactions, COMMAND_QUEUE_PARALLELISM * 0.3); + const transactionQueueLength = this.blockchainModuleManager.getTransactionQueueLength(blockchain); if (transactionQueueLength >= totalTransactions) return; @@ -75,8 +85,8 @@ class EpochCheckCommand extends Command { updateServiceAgreementsLastCommitEpoch.push( this.repositoryModuleManager.updateServiceAgreementLastCommitEpoch( - serviceAgreement.agreement_id, - serviceAgreement.current_epoch, + serviceAgreement.agreementId, + serviceAgreement.currentEpoch, ), ); @@ -165,7 +175,7 @@ class EpochCheckCommand extends Command { ); const peerId = this.networkModuleManager.getPeerId().toB58String(); - if (!neighbourhood.some((node) => node.peer_id === peerId)) { + if (!neighbourhood.some((node) => node.peerId === peerId)) { return; } @@ -207,6 +217,7 @@ class EpochCheckCommand extends Command { async scheduleSubmitCommitCommand(agreement) { const commandData = { + operationId: uuidv4(), blockchain: agreement.blockchainId, contract: agreement.assetStorageContractAddress, tokenId: agreement.tokenId, @@ -228,6 +239,7 @@ class EpochCheckCommand extends Command { async scheduleSubmitProofsCommand(agreement) { const commandData = { + operationId: uuidv4(), blockchain: agreement.blockchainId, contract: agreement.assetStorageContractAddress, tokenId: agreement.tokenId, diff --git a/src/commands/protocols/common/submit-commit-command.js b/src/commands/protocols/common/submit-commit-command.js index f32f5ad5d..7c9cfba13 100644 --- a/src/commands/protocols/common/submit-commit-command.js +++ b/src/commands/protocols/common/submit-commit-command.js @@ -1,4 +1,3 @@ -import { v4 as uuidv4 } from 'uuid'; import { OPERATION_ID_STATUS, ERROR_TYPE, COMMAND_RETRIES } from '../../../constants/constants.js'; import Command from '../../command.js'; @@ -14,6 +13,7 @@ class SubmitCommitCommand extends Command { async execute(command) { const { + operationId, blockchain, contract, tokenId, @@ -24,9 +24,6 @@ class SubmitCommitCommand extends Command { stateIndex, } = command.data; - // TODO: review operationId - const operationId = uuidv4(); - this.logger.trace( `Started ${command.name} for agreement id: ${agreementId} ` + `blockchain: ${blockchain}, contract: ${contract}, token id: ${tokenId},` + @@ -58,65 +55,38 @@ class SubmitCommitCommand extends Command { return Command.empty(); } - await this.blockchainModuleManager.submitCommit( - blockchain, - contract, - tokenId, - keyword, - hashFunctionId, - epoch, - stateIndex, - async (result) => { - if (!result.error) { - this.logger.trace( - `Successfully executed ${command.name} for agreement id: ${agreementId} ` + - `contract: ${contract}, token id: ${tokenId}, keyword: ${keyword}, ` + - `hash function id: ${hashFunctionId}. Retry number ${ - COMMAND_RETRIES.SUBMIT_COMMIT - command.retries + 1 - }`, - ); - this.operationIdService.emitChangeEvent( - OPERATION_ID_STATUS.COMMIT_PROOF.SUBMIT_COMMIT_END, - operationId, - agreementId, - epoch, - ); - } else if (command.retries - 1 === 0) { - const errorMessage = `Failed executing submit commit command, maximum number of retries reached. Error: ${result.error.message}.`; - this.logger.error(errorMessage); - this.operationIdService.emitChangeEvent( - OPERATION_ID_STATUS.FAILED, - operationId, - errorMessage, - this.errorType, - epoch, - ); - } else { - const blockTime = this.blockchainModuleManager.getBlockTimeMillis(blockchain); - this.logger.warn( - `Failed executing submit commit command, retrying in ${blockTime}ms. Error: ${result.error.message}`, - ); - await this.commandExecutor.add({ - name: 'submitCommitCommand', - sequence: [], - delay: blockTime, - retries: command.retries - 1, - data: command.data, - transactional: false, - }); - } - }, - ); + const transactionCompletePromise = new Promise((resolve, reject) => { + this.blockchainModuleManager.submitCommit( + blockchain, + contract, + tokenId, + keyword, + hashFunctionId, + epoch, + stateIndex, + (result) => { + if (result?.error) { + reject(result.error); + } + resolve(); + }, + ); + }); - const transactionQueueLength = - this.blockchainModuleManager.getTransactionQueueLength(blockchain); + await transactionCompletePromise; this.logger.trace( - `Scheduled submit commit transaction for agreement id: ${agreementId} ` + - `blockchain: ${blockchain}, contract: ${contract}, token id: ${tokenId},` + - `keyword: ${keyword}, hash function id: ${hashFunctionId}, epoch: ${epoch}, ` + - `stateIndex: ${stateIndex}, operationId: ${operationId}, ` + - `transaction queue length: ${transactionQueueLength}.`, + `Successfully executed ${command.name} for agreement id: ${agreementId} ` + + `contract: ${contract}, token id: ${tokenId}, keyword: ${keyword}, ` + + `hash function id: ${hashFunctionId}. Retry number ${ + COMMAND_RETRIES.SUBMIT_COMMIT - command.retries + 1 + }`, + ); + this.operationIdService.emitChangeEvent( + OPERATION_ID_STATUS.COMMIT_PROOF.SUBMIT_COMMIT_END, + operationId, + agreementId, + epoch, ); return Command.empty(); @@ -140,6 +110,10 @@ class SubmitCommitCommand extends Command { return false; } + async retryFinished(command) { + this.recover(command, `Max retry count for command: ${command.name} reached!`); + } + /** * Builds default handleStoreInitCommand * @param map diff --git a/src/commands/protocols/common/submit-proofs-command.js b/src/commands/protocols/common/submit-proofs-command.js index 16fef93d1..8348480f8 100644 --- a/src/commands/protocols/common/submit-proofs-command.js +++ b/src/commands/protocols/common/submit-proofs-command.js @@ -1,4 +1,3 @@ -import { v4 as uuidv4 } from 'uuid'; import { OPERATION_ID_STATUS, ERROR_TYPE, @@ -23,6 +22,7 @@ class SubmitProofsCommand extends Command { async execute(command) { const { + operationId, blockchain, contract, tokenId, @@ -34,9 +34,6 @@ class SubmitProofsCommand extends Command { stateIndex, } = command.data; - // TODO: review operationId - const operationId = uuidv4(); - this.logger.trace( `Started ${command.name} for agreement id: ${agreementId} ` + `blockchain: ${blockchain}, contract: ${contract}, token id: ${tokenId},` + @@ -92,83 +89,55 @@ class SubmitProofsCommand extends Command { agreementId, epoch, ); - } else { - const alreadySubmitted = await this.proofAlreadySubmitted( - blockchain, - agreementId, - epoch, - stateIndex, - ); - if (alreadySubmitted) { - this.logger.trace( - `Proofs already submitted for blockchain: ${blockchain} agreement id: ${agreementId}, epoch: ${epoch}, state index: ${stateIndex}`, - ); - return Command.empty(); - } } - - await this.blockchainModuleManager.sendProof( + const alreadySubmitted = await this.proofAlreadySubmitted( blockchain, - contract, - tokenId, - keyword, - hashFunctionId, + agreementId, epoch, - proof, - leaf, stateIndex, - async (result) => { - if (!result.error) { - this.logger.trace( - `Successfully executed ${command.name} for agreement id: ${agreementId} ` + - `contract: ${contract}, token id: ${tokenId}, keyword: ${keyword}, ` + - `hash function id: ${hashFunctionId}. Retry number ${ - COMMAND_RETRIES.SUBMIT_PROOFS - command.retries + 1 - }`, - ); - - this.operationIdService.emitChangeEvent( - OPERATION_ID_STATUS.COMMIT_PROOF.SUBMIT_PROOFS_END, - operationId, - agreementId, - epoch, - ); - } else if (command.retries - 1 === 0) { - const errorMessage = `Failed executing submit proofs command, maximum number of retries reached. Error: ${result.error.message}. Scheduling next epoch check.`; - this.logger.error(errorMessage); - this.operationIdService.emitChangeEvent( - OPERATION_ID_STATUS.FAILED, - operationId, - errorMessage, - this.errorType, - epoch, - ); - } else { - const blockTime = this.blockchainModuleManager.getBlockTimeMillis(blockchain); - this.logger.warn( - `Failed executing submit proofs command, retrying in ${blockTime}ms. Error: ${result.error.message}`, - ); - await this.commandExecutor.add({ - name: 'submitProofsCommand', - sequence: [], - delay: blockTime, - data: command.data, - retries: command.retries - 1, - transactional: false, - }); - } - }, ); + if (alreadySubmitted) { + this.logger.trace( + `Proofs already submitted for blockchain: ${blockchain} agreement id: ${agreementId}, epoch: ${epoch}, state index: ${stateIndex}`, + ); + return Command.empty(); + } + + const transactionCompletePromise = new Promise((resolve, reject) => { + this.blockchainModuleManager.sendProof( + blockchain, + contract, + tokenId, + keyword, + hashFunctionId, + epoch, + proof, + leaf, + stateIndex, + (result) => { + if (result?.error) { + reject(result.error); + } + resolve(); + }, + ); + }); - const transactionQueueLength = - this.blockchainModuleManager.getTransactionQueueLength(blockchain); + await transactionCompletePromise; this.logger.trace( - `Scheduled send proof transaction for agreement id: ${agreementId} ` + - `blockchain: ${blockchain}, contract: ${contract}, token id: ${tokenId},` + - `keyword: ${keyword}, hash function id: ${hashFunctionId}, epoch: ${epoch}, ` + - `stateIndex: ${stateIndex}, operationId: ${operationId}, ` + - `transaction queue length: ${transactionQueueLength}.`, + `Successfully executed ${command.name} for agreement id: ${agreementId} ` + + `contract: ${contract}, token id: ${tokenId}, keyword: ${keyword}, ` + + `hash function id: ${hashFunctionId}. Retry number ${ + COMMAND_RETRIES.SUBMIT_PROOFS - command.retries + 1 + }`, + ); + + this.operationIdService.emitChangeEvent( + OPERATION_ID_STATUS.COMMIT_PROOF.SUBMIT_PROOFS_END, + operationId, + agreementId, + epoch, ); return Command.empty(); @@ -192,6 +161,10 @@ class SubmitProofsCommand extends Command { return false; } + async retryFinished(command) { + this.recover(command, `Max retry count for command: ${command.name} reached!`); + } + /** * Builds default submitProofsCommand * @param map diff --git a/src/commands/protocols/update/receiver/submit-update-commit-command.js b/src/commands/protocols/update/receiver/submit-update-commit-command.js index 682e16aa2..7c40eb0f5 100644 --- a/src/commands/protocols/update/receiver/submit-update-commit-command.js +++ b/src/commands/protocols/update/receiver/submit-update-commit-command.js @@ -60,7 +60,7 @@ class SubmitUpdateCommitCommand extends Command { return Command.empty(); } - await this.blockchainModuleManager.submitUpdateCommit( + this.blockchainModuleManager.submitUpdateCommit( blockchain, contract, tokenId, diff --git a/src/commands/protocols/update/receiver/v1.0.0/v1-0-0-handle-update-request-command.js b/src/commands/protocols/update/receiver/v1.0.0/v1-0-0-handle-update-request-command.js index 380ecdd76..c26f461a4 100644 --- a/src/commands/protocols/update/receiver/v1.0.0/v1-0-0-handle-update-request-command.js +++ b/src/commands/protocols/update/receiver/v1.0.0/v1-0-0-handle-update-request-command.js @@ -152,7 +152,7 @@ class HandleUpdateRequestCommand extends HandleProtocolMessageCommand { ); const peerId = this.networkModuleManager.getPeerId().toB58String(); - if (!neighbourhood.some((node) => node.peer_id === peerId)) { + if (!neighbourhood.some((node) => node.peerId === peerId)) { return; } diff --git a/src/constants/constants.js b/src/constants/constants.js index 2abc0abae..611ded831 100644 --- a/src/constants/constants.js +++ b/src/constants/constants.js @@ -351,6 +351,12 @@ export const COMMAND_STATUS = { REPEATING: 'REPEATING', }; +/** + * How many commands will run in parallel + * @type {number} + */ +export const COMMAND_QUEUE_PARALLELISM = 100; + /** * @constant {object} NETWORK_PROTOCOLS - * Network protocols diff --git a/src/modules/blockchain/blockchain-module-manager.js b/src/modules/blockchain/blockchain-module-manager.js index 182108b11..2079c1327 100644 --- a/src/modules/blockchain/blockchain-module-manager.js +++ b/src/modules/blockchain/blockchain-module-manager.js @@ -236,7 +236,7 @@ class BlockchainModuleManager extends BaseModuleManager { ]); } - async submitCommit( + submitCommit( blockchain, assetContractAddress, tokenId, @@ -257,7 +257,7 @@ class BlockchainModuleManager extends BaseModuleManager { ]); } - async submitUpdateCommit( + submitUpdateCommit( blockchain, assetContractAddress, tokenId, @@ -293,7 +293,7 @@ class BlockchainModuleManager extends BaseModuleManager { ]); } - async sendProof( + sendProof( blockchain, assetContractAddress, tokenId, diff --git a/src/modules/blockchain/implementation/web3-service.js b/src/modules/blockchain/implementation/web3-service.js index 42e4c866b..54f122b55 100644 --- a/src/modules/blockchain/implementation/web3-service.js +++ b/src/modules/blockchain/implementation/web3-service.js @@ -70,7 +70,7 @@ class Web3Service { }, concurrency); } - async queueTransaction(contractInstance, functionName, transactionArgs, callback) { + queueTransaction(contractInstance, functionName, transactionArgs, callback) { this.transactionQueue.push( { contractInstance, @@ -635,7 +635,7 @@ class Web3Service { return finalizationCommitsNumber; } - async submitCommit( + submitCommit( assetContractAddress, tokenId, keyword, @@ -652,14 +652,7 @@ class Web3Service { ); } - async submitUpdateCommit( - assetContractAddress, - tokenId, - keyword, - hashFunctionId, - epoch, - callback, - ) { + submitUpdateCommit(assetContractAddress, tokenId, keyword, hashFunctionId, epoch, callback) { return this.queueTransaction( this.CommitManagerV1U1Contract, 'submitUpdateCommit', @@ -695,7 +688,7 @@ class Web3Service { return { assertionId: result['0'], challenge: result['1'] }; } - async sendProof( + sendProof( assetContractAddress, tokenId, keyword, From 0805dce6be3274f85d6c7551e4376b442aa40389 Mon Sep 17 00:00:00 2001 From: zeroxbt <89495162+zeroxbt@users.noreply.github.com> Date: Mon, 22 May 2023 12:58:32 +0200 Subject: [PATCH 13/16] bump version to 6.0.9 (#2543) --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 0a55642e2..7d1bc7066 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "origintrail_node", - "version": "6.0.9+hotfix.2", + "version": "6.0.9", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "origintrail_node", - "version": "6.0.9+hotfix.2", + "version": "6.0.9", "license": "ISC", "dependencies": { "@comunica/query-sparql": "^2.4.3", diff --git a/package.json b/package.json index 6c27375d2..9a9ee6a17 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "origintrail_node", - "version": "6.0.9+hotfix.2", + "version": "6.0.9", "description": "OTNode V6", "main": "index.js", "type": "module", From 0397174aec9c686f4e4db89e8ba8d96276bc5dff Mon Sep 17 00:00:00 2001 From: zeroxbt <89495162+zeroxbt@users.noreply.github.com> Date: Mon, 22 May 2023 14:18:14 +0200 Subject: [PATCH 14/16] dont overwrite last commit, proof checks on update (#2548) --- .../repositories/service-agreement-repository.js | 8 ++++++++ src/modules/repository/repository-module-manager.js | 6 ++++++ src/service/blockchain-event-listener-service.js | 7 +++++++ 3 files changed, 21 insertions(+) diff --git a/src/modules/repository/implementation/sequelize/repositories/service-agreement-repository.js b/src/modules/repository/implementation/sequelize/repositories/service-agreement-repository.js index abdd54a0c..fdca41bf7 100644 --- a/src/modules/repository/implementation/sequelize/repositories/service-agreement-repository.js +++ b/src/modules/repository/implementation/sequelize/repositories/service-agreement-repository.js @@ -63,6 +63,14 @@ class ServiceAgreementRepository { }); } + async getServiceAgreementRecord(agreementId) { + return this.model.findOne({ + where: { + agreementId, + }, + }); + } + async updateServiceAgreementLastCommitEpoch(agreementId, lastCommitEpoch) { return this.model.update( { lastCommitEpoch }, diff --git a/src/modules/repository/repository-module-manager.js b/src/modules/repository/repository-module-manager.js index c331516d8..ffef9958e 100644 --- a/src/modules/repository/repository-module-manager.js +++ b/src/modules/repository/repository-module-manager.js @@ -301,6 +301,12 @@ class RepositoryModuleManager extends BaseModuleManager { } } + async getServiceAgreementRecord(agreementId) { + if (this.initialized) { + return this.getRepository('service_agreement').getServiceAgreementRecord(agreementId); + } + } + async updateServiceAgreementLastCommitEpoch(agreementId, lastCommitEpoch) { if (this.initialized) { return this.getRepository('service_agreement').updateServiceAgreementLastCommitEpoch( diff --git a/src/service/blockchain-event-listener-service.js b/src/service/blockchain-event-listener-service.js index 52fe36db7..a6a458139 100644 --- a/src/service/blockchain-event-listener-service.js +++ b/src/service/blockchain-event-listener-service.js @@ -501,6 +501,11 @@ class BlockchainEventListenerService { cachedData.agreementId && cachedData.agreementData ) { + const serviceAgreement = + await this.repositoryModuleManager.getServiceAgreementRecord( + cachedData.agreementId, + ); + await this.repositoryModuleManager.updateServiceAgreementRecord( blockchain, contract, @@ -515,6 +520,8 @@ class BlockchainEventListenerService { keyword, assertionId, stateIndex, + serviceAgreement?.lastCommitEpoch, + serviceAgreement?.lastProofEpoch, ); } } else if (currentRepository === TRIPLE_STORE_REPOSITORIES.PUBLIC_CURRENT) { From b7f4c5796c1d9ff4517f5c411b29fdd8fd192412 Mon Sep 17 00:00:00 2001 From: zeroxbt <89495162+zeroxbt@users.noreply.github.com> Date: Mon, 22 May 2023 14:20:25 +0200 Subject: [PATCH 15/16] bump version to 6.0.9+hotfix.3 (#2549) --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 7d1bc7066..78f4b5884 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "origintrail_node", - "version": "6.0.9", + "version": "6.0.9+hotfix.3", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "origintrail_node", - "version": "6.0.9", + "version": "6.0.9+hotfix.3", "license": "ISC", "dependencies": { "@comunica/query-sparql": "^2.4.3", diff --git a/package.json b/package.json index 9a9ee6a17..58a36de4e 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "origintrail_node", - "version": "6.0.9", + "version": "6.0.9+hotfix.3", "description": "OTNode V6", "main": "index.js", "type": "module", From 2c685372c318a931c4a223ca1424fb45edf13db0 Mon Sep 17 00:00:00 2001 From: zeroxbt <89495162+zeroxbt@users.noreply.github.com> Date: Mon, 22 May 2023 15:16:27 +0200 Subject: [PATCH 16/16] bump version to 6.0.9 (#2554) --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 78f4b5884..7d1bc7066 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "origintrail_node", - "version": "6.0.9+hotfix.3", + "version": "6.0.9", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "origintrail_node", - "version": "6.0.9+hotfix.3", + "version": "6.0.9", "license": "ISC", "dependencies": { "@comunica/query-sparql": "^2.4.3", diff --git a/package.json b/package.json index 58a36de4e..9a9ee6a17 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "origintrail_node", - "version": "6.0.9+hotfix.3", + "version": "6.0.9", "description": "OTNode V6", "main": "index.js", "type": "module",