diff --git a/modules/Blockchain.js b/modules/Blockchain.js index 26e82e134..effef7d27 100644 --- a/modules/Blockchain.js +++ b/modules/Blockchain.js @@ -338,10 +338,18 @@ class Blockchain { return this.blockchain.getPurchase(purchaseId); } + async getPurchaseStatus(purchaseId) { + return this.blockchain.getPurchaseStatus(purchaseId); + } + async getPurchasedData(importId, wallet) { return this.blockchain.getPurchasedData(importId, wallet); } + async getPaymentStageInterval() { + return this.blockchain.getPaymentStageInterval(); + } + async initiatePurchase( sellerIdentity, buyerIdentity, tokenAmount, @@ -372,6 +380,26 @@ class Blockchain { return this.blockchain.takePayment(purchaseId); } + async complainAboutNode( + purchaseId, outputIndex, inputIndexLeft, encodedOutput, encodedInputLeft, + proofOfEncodedOutput, proofOfEncodedInputLeft, urgent, + ) { + return this.blockchain.complainAboutNode( + purchaseId, outputIndex, inputIndexLeft, encodedOutput, encodedInputLeft, + proofOfEncodedOutput, proofOfEncodedInputLeft, urgent, + ); + } + + async complainAboutRoot( + purchaseId, encodedRootHash, proofOfEncodedRootHash, rootHashIndex, + urgent, + ) { + return this.blockchain.complainAboutRoot( + purchaseId, encodedRootHash, proofOfEncodedRootHash, rootHashIndex, + urgent, + ); + } + async sendCommitment(importId, dvWallet, commitment) { return this.blockchain.sendCommitment(importId, dvWallet, commitment); } @@ -612,6 +640,15 @@ class Blockchain { async keyHasPurpose(identity, key, purpose) { return this.blockchain.keyHasPurpose(identity, key, purpose); } + + /** + * Check how many events were emitted in a transaction from the transaction receipt + * @param receipt - the json object returned as a result of the transaction + * @return {Number | undefined} - Returns undefined if the receipt does not have a logs field + */ + numberOfEventsEmitted(receipt) { + return this.blockchain.numberOfEventsEmitted(receipt); + } } module.exports = Blockchain; diff --git a/modules/Blockchain/Ethereum/abi/marketplace.json b/modules/Blockchain/Ethereum/abi/marketplace.json index a6889a8b6..368fa7cc0 100644 --- a/modules/Blockchain/Ethereum/abi/marketplace.json +++ b/modules/Blockchain/Ethereum/abi/marketplace.json @@ -11,7 +11,8 @@ ], "payable": false, "stateMutability": "view", - "type": "function" + "type": "function", + "signature": "0x365a86fc" }, { "constant": true, @@ -25,7 +26,8 @@ ], "payable": false, "stateMutability": "view", - "type": "function" + "type": "function", + "signature": "0x691c6deb" }, { "inputs": [ @@ -36,7 +38,8 @@ ], "payable": false, "stateMutability": "nonpayable", - "type": "constructor" + "type": "constructor", + "signature": "constructor" }, { "anonymous": false, @@ -73,7 +76,8 @@ } ], "name": "PurchaseInitiated", - "type": "event" + "type": "event", + "signature": "0xc93d5f3be3fb6a3a357ff8618f169d1fc7f2af9dfaa690855956a3be0a5a23fd" }, { "anonymous": false, @@ -90,7 +94,8 @@ } ], "name": "KeyDeposited", - "type": "event" + "type": "event", + "signature": "0x085125356d0e6a726b611531ed99af43bc381be83e7a55a9b0f111578236168b" }, { "anonymous": false, @@ -112,7 +117,31 @@ } ], "name": "MisbehaviourProven", - "type": "event" + "type": "event", + "signature": "0xf4be0903b752a94a903248b0bab983f94b443f9fcae393f657e06b05d5405b06" + }, + { + "anonymous": false, + "inputs": [ + { + "indexed": false, + "name": "purchaseId", + "type": "bytes32" + }, + { + "indexed": false, + "name": "sellerIdentity", + "type": "address" + }, + { + "indexed": false, + "name": "buyerIdentity", + "type": "address" + } + ], + "name": "PurchaseCompleted", + "type": "event", + "signature": "0x3c714883152f8c067c4b9eb1d7ebc8bf19da6a1999475e4842aba88c93943143" }, { "constant": false, @@ -126,7 +155,8 @@ "outputs": [], "payable": false, "stateMutability": "nonpayable", - "type": "function" + "type": "function", + "signature": "0xc7eb4522" }, { "constant": false, @@ -156,7 +186,8 @@ "outputs": [], "payable": false, "stateMutability": "nonpayable", - "type": "function" + "type": "function", + "signature": "0x940e6f29" }, { "constant": false, @@ -174,7 +205,8 @@ "outputs": [], "payable": false, "stateMutability": "nonpayable", - "type": "function" + "type": "function", + "signature": "0x10a2d0d7" }, { "constant": false, @@ -188,7 +220,8 @@ "outputs": [], "payable": false, "stateMutability": "nonpayable", - "type": "function" + "type": "function", + "signature": "0x2eff62c5" }, { "constant": false, @@ -214,7 +247,8 @@ "outputs": [], "payable": false, "stateMutability": "nonpayable", - "type": "function" + "type": "function", + "signature": "0x7a2af025" }, { "constant": false, @@ -252,7 +286,8 @@ "outputs": [], "payable": false, "stateMutability": "nonpayable", - "type": "function" + "type": "function", + "signature": "0x5c31aa65" }, { "constant": true, @@ -279,7 +314,8 @@ ], "payable": false, "stateMutability": "pure", - "type": "function" + "type": "function", + "signature": "0x92b775c6" }, { "constant": true, @@ -310,6 +346,7 @@ ], "payable": false, "stateMutability": "view", - "type": "function" + "type": "function", + "signature": "0x9e6731ad" } ] \ No newline at end of file diff --git a/modules/Blockchain/Ethereum/contracts/Marketplace.sol b/modules/Blockchain/Ethereum/contracts/Marketplace.sol index 2ef023194..6e487fc4c 100644 --- a/modules/Blockchain/Ethereum/contracts/Marketplace.sol +++ b/modules/Blockchain/Ethereum/contracts/Marketplace.sol @@ -47,6 +47,7 @@ contract Marketplace { ); event KeyDeposited(bytes32 purchaseId, bytes32 key); event MisbehaviourProven(bytes32 purchaseId, address sellerIdentity, address buyerIdentity); + event PurchaseCompleted(bytes32 purchaseId, address sellerIdentity, address buyerIdentity); function initiatePurchase( address sellerIdentity, @@ -130,6 +131,8 @@ contract Marketplace { marketplaceStorage.setStage(purchaseId, 3); marketplaceStorage.setTimestamp(purchaseId, block.timestamp); + + emit PurchaseCompleted(purchaseId, sellerIdentity, buyerIdentity); } function reserveTokens(address party, uint256 amount) diff --git a/modules/Blockchain/Ethereum/index.js b/modules/Blockchain/Ethereum/index.js index c56dccfa4..545320fa2 100644 --- a/modules/Blockchain/Ethereum/index.js +++ b/modules/Blockchain/Ethereum/index.js @@ -966,11 +966,22 @@ class Ethereum { return this.marketplaceStorageContract.methods.purchase(purchaseId).call(); } + async getPurchaseStatus(purchaseId) { + this.logger.trace(`Asking for purchase with id [${purchaseId}].`); + return this.marketplaceStorageContract.methods.getStage(purchaseId).call(); + } + async getPurchasedData(importId, wallet) { this.logger.trace(`Asking purchased data for import ${importId} and wallet ${wallet}.`); return this.readingContract.methods.purchased_data(importId, wallet).call(); } + async getPaymentStageInterval() { + this.logger.trace('Reading payment stage interval from blockchain.'); + return this.marketplaceContract.methods.paymentStageInterval().call(); + } + + async initiatePurchase( sellerIdentity, buyerIdentity, tokenAmount, @@ -1047,6 +1058,44 @@ class Ethereum { ); } + async complainAboutNode( + purchaseId, outputIndex, inputIndexLeft, encodedOutput, encodedInputLeft, + proofOfEncodedOutput, proofOfEncodedInputLeft, urgent, + ) { + const gasPrice = await this.getGasPrice(urgent); + const options = { + gasLimit: this.web3.utils.toHex(this.config.gas_limit), + gasPrice: this.web3.utils.toHex(gasPrice), + to: this.marketplaceContractAddress, + }; + + this.logger.trace(`complainAboutNode(${purchaseId},${outputIndex},${inputIndexLeft},` + + `${encodedOutput},${encodedInputLeft},${proofOfEncodedOutput},${proofOfEncodedInputLeft})`); + return this.transactions.queueTransaction( + this.marketplaceContractAbi, 'complainAboutNode', + [purchaseId, outputIndex, inputIndexLeft, encodedOutput, encodedInputLeft, + proofOfEncodedOutput, proofOfEncodedInputLeft], options, + ); + } + + async complainAboutRoot( + purchaseId, encodedRootHash, proofOfEncodedRootHash, rootHashIndex, + urgent, + ) { + const gasPrice = await this.getGasPrice(urgent); + const options = { + gasLimit: this.web3.utils.toHex(this.config.gas_limit), + gasPrice: this.web3.utils.toHex(gasPrice), + to: this.marketplaceContractAddress, + }; + + this.logger.trace(`complainAboutRoot(${purchaseId},${encodedRootHash},${proofOfEncodedRootHash},${rootHashIndex})`); + return this.transactions.queueTransaction( + this.marketplaceContractAbi, 'complainAboutRoot', + [purchaseId, encodedRootHash, proofOfEncodedRootHash, rootHashIndex], options, + ); + } + async sendCommitment(importId, dvWallet, commitment) { const gasPrice = await this.getGasPrice(); const options = { @@ -1547,6 +1596,18 @@ class Ethereum { return gasPrice; } } + + /** + * Check how many events were emitted in a transaction from the transaction receipt + * @param receipt - the json object returned as a result of the transaction + * @return {Number | undefined} - Returns undefined if the receipt does not have a logs field + */ + numberOfEventsEmitted(receipt) { + if (!receipt || !receipt.logs || !Array.isArray(receipt.logs)) { + return undefined; + } + return receipt.logs.length; + } } module.exports = Ethereum; diff --git a/modules/EventEmitter.js b/modules/EventEmitter.js index 042192ad4..ef8bc2f19 100644 --- a/modules/EventEmitter.js +++ b/modules/EventEmitter.js @@ -451,7 +451,7 @@ class EventEmitter { logger.info(`Request for data ${message.query[0].value} from DV ${message.wallet} received`); if (!Utilities.isMessageSigned(this.web3, message, messageSignature)) { - logger.warn(`We have a forger here. Signature doesn't match for message: ${message.toString()}`); + logger.warn(`We have a forger here. Signature doesn't match for message: ${JSON.stringify(message)}`); return; } @@ -469,6 +469,35 @@ class EventEmitter { } }); + this._on('kad-purchase-complete', async (request) => { + const { message, messageSignature } = request; + + logger.info(`Purchase confirmation from DV ${message.seller_node_id} received`); + + if (!Utilities.isMessageSigned(this.web3, message, messageSignature)) { + logger.warn(`We have a forger here. Signature doesn't match for message: ${JSON.stringify(message)}`); + return; + } + + try { + const { + purchase_id, + data_set_id, + ot_object_id, + seller_node_id, + seller_erc_id, + price, + } = message; + await dvController.handleNewDataSeller( + purchase_id, seller_erc_id, seller_node_id, + data_set_id, ot_object_id, price, + ); + } catch (error) { + const errorMessage = `Failed to process purchase completion message. ${error}.`; + logger.warn(errorMessage); + } + }); + // sync this._on('kad-replication-request', async (request, response) => { const kadReplicationRequest = transport.extractMessage(request); @@ -478,7 +507,7 @@ class EventEmitter { replicationMessage = message; if (!Utilities.isMessageSigned(this.web3, message, messageSignature)) { - logger.warn(`We have a forger here. Signature doesn't match for message: ${message.toString()}`); + logger.warn(`We have a forger here. Signature doesn't match for message: ${JSON.stringify(message)}`); return; } } @@ -550,7 +579,7 @@ class EventEmitter { message, messageSignature, } = replicationFinishedMessage; if (!Utilities.isMessageSigned(this.web3, message, messageSignature)) { - logger.warn(`We have a forger here. Signature doesn't match for message: ${message.toString()}`); + logger.warn(`We have a forger here. Signature doesn't match for message: ${JSON.stringify(message)}`); return; } @@ -609,7 +638,7 @@ class EventEmitter { if (challengeRequest.messageSignature) { // todo remove if for next update const { messageSignature } = challengeRequest; if (!Utilities.isMessageSigned(this.web3, message, messageSignature)) { - logger.warn(`We have a forger here. Signature doesn't match for message: ${message.toString()}`); + logger.warn(`We have a forger here. Signature doesn't match for message: ${JSON.stringify(message)}`); return; } @@ -654,7 +683,7 @@ class EventEmitter { if (challengeResponse.messageSignature) { // todo remove if for next update const { messageSignature } = challengeResponse; if (!Utilities.isMessageSigned(this.web3, message, messageSignature)) { - logger.warn(`We have a forger here. Signature doesn't match for message: ${message.toString()}`); + logger.warn(`We have a forger here. Signature doesn't match for message: ${JSON.stringify(message)}`); return; } @@ -682,7 +711,7 @@ class EventEmitter { const { message, messageSignature } = dataLocationResponseObject; if (!Utilities.isMessageSigned(this.web3, message, messageSignature)) { - const returnMessage = `We have a forger here. Signature doesn't match for message: ${message.toString()}`; + const returnMessage = `We have a forger here. Signature doesn't match for message: ${JSON.stringify(message)}`; logger.warn(returnMessage); return; } @@ -701,8 +730,8 @@ class EventEmitter { const { message, messageSignature } = dataReadRequestObject; if (!Utilities.isMessageSigned(this.web3, message, messageSignature)) { - logger.warn(`We have a forger here. Signature doesn't match for message: ${message.toString()}`); - const returnMessage = `We have a forger here. Signature doesn't match for message: ${message.toString()}`; + logger.warn(`We have a forger here. Signature doesn't match for message: ${JSON.stringify(message)}`); + const returnMessage = `We have a forger here. Signature doesn't match for message: ${JSON.stringify(message)}`; logger.warn(returnMessage); return; } @@ -730,7 +759,7 @@ class EventEmitter { if (!Utilities.isMessageSigned(this.web3, message, messageSignature)) { console.log('kad-data-read-response', JSON.stringify(message), JSON.stringify(messageSignature)); - logger.warn(`We have a forger here. Signature doesn't match for message: ${message.toString()}`); + logger.warn(`We have a forger here. Signature doesn't match for message: ${JSON.stringify(message)}`); return; } @@ -747,7 +776,7 @@ class EventEmitter { const { message, messageSignature } = dataReadRequestObject; if (!Utilities.isMessageSigned(this.web3, message, messageSignature)) { - logger.warn(`We have a forger here. Signature doesn't match for message: ${message.toString()}`); + logger.warn(`We have a forger here. Signature doesn't match for message: ${JSON.stringify(message)}`); return; } try { @@ -765,7 +794,7 @@ class EventEmitter { const { message, messageSignature } = dataReadRequestObject; if (!Utilities.isMessageSigned(this.web3, message, messageSignature)) { - logger.warn(`We have a forger here. Signature doesn't match for message: ${message.toString()}`); + logger.warn(`We have a forger here. Signature doesn't match for message: ${JSON.stringify(message)}`); return; } try { @@ -788,7 +817,7 @@ class EventEmitter { const { message, messageSignature } = reqMessage; if (!Utilities.isMessageSigned(this.web3, message, messageSignature)) { - logger.warn(`We have a forger here. Signature doesn't match for message: ${message.toString()}`); + logger.warn(`We have a forger here. Signature doesn't match for message: ${JSON.stringify(message)}`); return; } @@ -813,7 +842,7 @@ class EventEmitter { const { message, messageSignature } = reqMessage; if (!Utilities.isMessageSigned(this.web3, message, messageSignature)) { - logger.warn(`We have a forger here. Signature doesn't match for message: ${message.toString()}`); + logger.warn(`We have a forger here. Signature doesn't match for message: ${JSON.stringify(message)}`); return; } @@ -838,7 +867,7 @@ class EventEmitter { const { message, messageSignature } = reqMessage; if (!Utilities.isMessageSigned(this.web3, message, messageSignature)) { - logger.warn(`We have a forger here. Signature doesn't match for message: ${message.toString()}`); + logger.warn(`We have a forger here. Signature doesn't match for message: ${JSON.stringify(message)}`); return; } @@ -863,7 +892,7 @@ class EventEmitter { const { message, messageSignature } = reqMessage; if (!Utilities.isMessageSigned(this.web3, message, messageSignature)) { - logger.warn(`We have a forger here. Signature doesn't match for message: ${message.toString()}`); + logger.warn(`We have a forger here. Signature doesn't match for message: ${JSON.stringify(message)}`); return; } @@ -883,7 +912,7 @@ class EventEmitter { const { message, messageSignature } = encryptedPaddedKeyObject; if (!Utilities.isMessageSigned(this.web3, message, messageSignature)) { - logger.warn(`We have a forger here. Signature doesn't match for message: ${message.toString()}`); + logger.warn(`We have a forger here. Signature doesn't match for message: ${JSON.stringify(message)}`); return; } diff --git a/modules/ImportUtilities.js b/modules/ImportUtilities.js index 839709672..e8c7d5154 100644 --- a/modules/ImportUtilities.js +++ b/modules/ImportUtilities.js @@ -206,9 +206,9 @@ class ImportUtilities { * @param permissioned_object * @returns {null} */ - static calculatePermissionedDataHash(permissioned_object, type = 'distribution') { + static calculatePermissionedDataHash(permissioned_object) { const merkleTree = ImportUtilities - .calculatePermissionedDataMerkleTree(permissioned_object, type); + .calculatePermissionedDataMerkleTree(permissioned_object); return merkleTree.getRoot(); } @@ -219,7 +219,7 @@ class ImportUtilities { * @param permissioned_object * @returns {null} */ - static calculatePermissionedDataMerkleTree(permissioned_object, type = 'distribution') { + static calculatePermissionedDataMerkleTree(permissioned_object) { if (!permissioned_object || !permissioned_object.data) { throw Error('Cannot calculate root hash of an empty object'); } @@ -237,7 +237,8 @@ class ImportUtilities { const block = data.slice(i, i + block_size).toString('hex'); blocks.push(block.padStart(64, '0')); } - const merkleTree = new MerkleTree(blocks, type, 'sha3'); + + const merkleTree = new MerkleTree(blocks, 'purchase', 'soliditySha3'); return merkleTree; } diff --git a/modules/command/dh/dh-data-read-request-free-command.js b/modules/command/dh/dh-data-read-request-free-command.js index 3ef995481..a1c458e0f 100644 --- a/modules/command/dh/dh-data-read-request-free-command.js +++ b/modules/command/dh/dh-data-read-request-free-command.js @@ -59,6 +59,19 @@ class DHDataReadRequestFreeCommand extends Command { throw Error(`Failed to get data info for import ID ${importId}.`); } + const allowedPermissionedDataElements = await Models.data_trades.findAll({ + where: { + data_set_id: importId, + buyer_node_id: nodeId, + status: 'COMPLETED', + }, + }); + + const privateData = {}; + + allowedPermissionedDataElements.forEach(element => + privateData[element.ot_json_object_id] = {}); + const document = await this.importService.getImport(importId); const permissionedData = await this.permissionedDataService.getAllowedPermissionedData( diff --git a/modules/command/dh/dh-pay-out-command.js b/modules/command/dh/dh-pay-out-command.js index 92006002c..d1db96052 100644 --- a/modules/command/dh/dh-pay-out-command.js +++ b/modules/command/dh/dh-pay-out-command.js @@ -75,7 +75,20 @@ class DhPayOutCommand extends Command { } catch (error) { if (error.message.includes('Gas price higher than maximum allowed price')) { this.logger.info('Gas price too high, delaying call for 30 minutes'); - return Command.repeat(); + return { + commands: [ + { + name: 'dhPayOutCommand', + delay: constants.GAS_PRICE_VALIDITY_TIME_IN_MILLS, + retries: 3, + transactional: false, + data: { + offerId, + viaAPI: false, + }, + }, + ], + }; } throw error; } diff --git a/modules/command/dh/dh-purchase-initiated-command.js b/modules/command/dh/dh-purchase-initiated-command.js index 49fd9ac7c..0bbbfd181 100644 --- a/modules/command/dh/dh-purchase-initiated-command.js +++ b/modules/command/dh/dh-purchase-initiated-command.js @@ -72,10 +72,15 @@ class DhPurchaseInitiatedCommand extends Command { purchase_id: purchaseId, }; + let delay = await this.blockchain.getPaymentStageInterval(); + delay = parseInt(delay, 10) * 1000; + + this.logger.info(`Key deposited for purchaseID ${purchaseId}.` + + ' Waiting for complaint window to expire before taking payment.'); await this.commandExecutor.add({ name: 'dhPurchaseTakePaymentCommand', data: commandData, - delay: 5 * 60 * 1000, + delay, retries: 3, }); return Command.empty(); diff --git a/modules/command/dh/dh-purchase-requested-command.js b/modules/command/dh/dh-purchase-requested-command.js index e092bf451..8bc64ffb2 100644 --- a/modules/command/dh/dh-purchase-requested-command.js +++ b/modules/command/dh/dh-purchase-requested-command.js @@ -92,7 +92,7 @@ class DhPurchaseRequestedCommand extends Command { }; await this.commandExecutor.add({ name: 'dhPurchaseInitiatedCommand', - delay: 1 * 60 * 1000, // todo check why is this necessary + delay: 60 * 1000, retries: 3, data: commandData, }); @@ -112,9 +112,13 @@ class DhPurchaseRequestedCommand extends Command { } } - this.logger.info(`Purchase confirmed for ot_object ${ot_json_object_id} received from ${dv_node_id}. Sending purchase response.`); + if (response.status === 'FAILED') { + this.logger.warn(`Failed to confirm purchase request. ${response.message}`); + } else { + this.logger.info(`Purchase confirmed for ot_object ${ot_json_object_id} received from ${dv_node_id}. Sending purchase response.`); + } await this._sendResponseToDv(response, dv_node_id); - + this.logger.info(`Purchase request response sent to ${dv_node_id}.`); return Command.empty(); } diff --git a/modules/command/dh/dh-purchase-take-payment-command.js b/modules/command/dh/dh-purchase-take-payment-command.js index 492e890d9..08b1cf43b 100644 --- a/modules/command/dh/dh-purchase-take-payment-command.js +++ b/modules/command/dh/dh-purchase-take-payment-command.js @@ -38,8 +38,9 @@ class DhPurchaseTakePaymentCommand extends Command { seller_erc_id: dataTrade.seller_erc_id, price: dataTrade.price, }); + this.logger.info(`Payment has been taken for purchase ${purchase_id}`); } catch (error) { - if (error.message.contains('Complaint window has not yet expired!')) { + if (error.message.includes('Complaint window has not yet expired!')) { if (command.retries !== 0) { return Command.retry(); } @@ -57,7 +58,10 @@ class DhPurchaseTakePaymentCommand extends Command { dataTrade.status = 'DISPUTED'; await dataTrade.save({ fields: ['status'] }); - this.logger.warn(`Couldn't take payment for purchase ${purchase_id}`); + await this._handleError( + purchase_id, + `Couldn't execute takePayment command for purchase with purchaseId ${purchase_id}. Error: Data mismatch proven in dispute`, + ); } return Command.empty(); diff --git a/modules/command/dv/dv-permissioned-data-read-request-command.js b/modules/command/dv/dv-permissioned-data-read-request-command.js index b9a19df6b..0f0d0eb8f 100644 --- a/modules/command/dv/dv-permissioned-data-read-request-command.js +++ b/modules/command/dv/dv-permissioned-data-read-request-command.js @@ -25,7 +25,7 @@ class DVPermissionedDataReadRequestCommand extends Command { data_set_id, ot_object_id, seller_node_id, - handlerId, + handler_id, } = command.data; const message = { @@ -34,7 +34,7 @@ class DVPermissionedDataReadRequestCommand extends Command { wallet: this.config.node_wallet, nodeId: this.config.identity, dv_erc725_identity: this.config.erc725Identity, - handler_id: handlerId, + handler_id, }; const dataReadRequestObject = { message, @@ -45,7 +45,7 @@ class DVPermissionedDataReadRequestCommand extends Command { ), }; - await this.transport.sendPrivateDataReadRequest( + await this.transport.sendPermissionedDataReadRequest( dataReadRequestObject, seller_node_id, ); @@ -56,7 +56,7 @@ class DVPermissionedDataReadRequestCommand extends Command { /** * Recover system from failure * @param command - * @param err + * @param error */ async recover(command, error) { const { handler_id } = command.data; diff --git a/modules/command/dv/dv-purchase-dispute-command.js b/modules/command/dv/dv-purchase-dispute-command.js index 31cbf53db..aaf7849dc 100644 --- a/modules/command/dv/dv-purchase-dispute-command.js +++ b/modules/command/dv/dv-purchase-dispute-command.js @@ -1,5 +1,9 @@ const Command = require('../command'); const Models = require('../../../models'); +const Utilities = require('../../Utilities'); +const constants = require('../../constants'); + +const { Op } = Models.Sequelize; /** * Handles data location response. @@ -7,8 +11,10 @@ const Models = require('../../../models'); class DvPurchaseDisputeCommand extends Command { constructor(ctx) { super(ctx); + this.blockchain = ctx.blockchain; this.logger = ctx.logger; this.remoteControl = ctx.remoteControl; + this.permissionedDataService = ctx.permissionedDataService; } /** @@ -18,8 +24,99 @@ class DvPurchaseDisputeCommand extends Command { */ async execute(command, transaction) { // send dispute purchase to bc + const { + handler_id, + encoded_data, + key, + purchase_id, + permissioned_data_array_length, + permissioned_data_original_length, + error_type, + } = command.data; + + try { + const purchaseStatus = await this.blockchain.getPurchaseStatus(purchase_id); + + if (purchaseStatus !== '2') { + throw new Error(`Cannot issue complaint for purchaseId ${purchase_id}. Purchase already completed`); + } + + this.remoteControl.purchaseStatus('Purchase not confirmed', 'Sending dispute purchase to Blockchain.', true); + + this.logger.important(`Initiating complaint for purchaseId ${purchase_id}`); + + let result; + if (error_type === constants.PURCHASE_ERROR_TYPE.NODE_ERROR) { + const { + input_index_left, + output_index, + } = command.data; + + const { + encodedInputLeft, + encodedOutput, + proofOfEncodedInputLeft, + proofOfEncodedOutput, + } = this.permissionedDataService.prepareNodeDisputeData( + encoded_data, + input_index_left, + output_index, + ); + + result = await this.blockchain.complainAboutNode( + Utilities.normalizeHex(purchase_id), + output_index, + input_index_left, + Utilities.normalizeHex(encodedOutput), + Utilities.normalizeHex(encodedInputLeft), + proofOfEncodedOutput, + proofOfEncodedInputLeft, + true, + ); + } else if (error_type === constants.PURCHASE_ERROR_TYPE.ROOT_ERROR) { + const { + rootHashIndex, + encodedRootHash, + proofOfEncodedRootHash, + } = this.permissionedDataService.prepareRootDisputeData(encoded_data); + + result = await this.blockchain.complainAboutRoot( + Utilities.normalizeHex(purchase_id), + Utilities.normalizeHex(encodedRootHash), + proofOfEncodedRootHash, + rootHashIndex, + true, + ); + } + + if (this.blockchain.numberOfEventsEmitted(result) >= 1) { + this.logger.important(`Purchase complaint for purchaseId ${purchase_id} approved. Refund received.`); + } else { + throw new Error(`Purchase complaint for purchaseId ${purchase_id} rejected.`); + } + } catch (error) { + await this._handleError(purchase_id, error.message); + } + + return Command.empty(); + } - this.remoteControl.purchaseStatus('Purchase not confirmed', 'Sending dispute purchase to Blockchain.', true); + async _handleError( + purchase_id, + errorMessage, + ) { + this.logger.error(errorMessage); + await Models.data_trades.update( + { + status: 'FAILED', + }, + { + where: { + purchase_id, + status: { [Op.ne]: 'FAILED' }, + }, + }, + ); } /** diff --git a/modules/command/dv/dv-purchase-initiate-command.js b/modules/command/dv/dv-purchase-initiate-command.js index abc0923d6..232e84ef2 100644 --- a/modules/command/dv/dv-purchase-initiate-command.js +++ b/modules/command/dv/dv-purchase-initiate-command.js @@ -1,4 +1,5 @@ const Command = require('../command'); +const MerkleTree = require('../../Merkle'); const Models = require('../../../models'); const { Op } = Models.Sequelize; @@ -30,8 +31,8 @@ class DvPurchaseInitiateCommand extends Command { if (status !== 'SUCCESSFUL') { - this.logger.trace(`Unable to initiate purchase, seller returned status: ${status} with message: ${message}`); - this._handleError(handler_id, status); + this.logger.warn(`Unable to initiate purchase, seller returned status: ${status} with message: ${message}`); + await this._handleError(handler_id, status); return Command.empty(); } const { @@ -39,17 +40,31 @@ class DvPurchaseInitiateCommand extends Command { seller_node_id, ot_object_id, } = await this._getHandlerData(handler_id); + this.logger.trace(`Received encoded permissioned data for object ${ot_object_id} from seller ${seller_node_id}. Verifying data integrity...`); const permissionedObject = await this.importService.getOtObjectById( data_set_id, ot_object_id, ); - if (permissioned_data_root_hash !== permissionedObject.permissioned_data_hash) { - this._handleError(handler_id, 'Unable to initiate purchase. Permissioned data root hash validation failed'); + if (permissioned_data_root_hash !== + permissionedObject.properties.permissioned_data.permissioned_data_hash) { + await this._handleError(handler_id, 'Unable to initiate purchase. Permissioned data root hash validation failed'); return Command.empty(); } + // Verify data integrity + // Recreate merkle tree + const encodedMerkleTree = new MerkleTree(encoded_data, 'purchase', 'soliditySha3'); + const encodedDataRootHash = encodedMerkleTree.getRoot(); + + if (encoded_data_root_hash !== encodedDataRootHash) { + await this._handleError(handler_id, 'Unable to initiate purchase. Encoded data root hash validation failed'); + return Command.empty(); + } + + this.logger.info('Purchase response verified. Initiating purchase on the blockchain...'); + const dataTrade = await Models.data_trades.findOne({ where: { data_set_id, @@ -60,17 +75,15 @@ class DvPurchaseInitiateCommand extends Command { }); const result = await this.blockchain.initiatePurchase( dataTrade.seller_erc_id, dataTrade.buyer_erc_id, - dataTrade.price, - permissioned_data_root_hash, encoded_data_root_hash, + dataTrade.price, permissioned_data_root_hash, encoded_data_root_hash, ); - const { purchaseId } = this.blockchain - .decodePurchaseInitiatedEventFromTransaction(result); + const { purchaseId } = this.blockchain.decodePurchaseInitiatedEventFromTransaction(result); this.logger.important(`Purchase ${purchaseId} initiated. Waiting for key from seller...`); if (!purchaseId) { this.remoteControl.purchaseStatus('Purchase failed', 'Unabled to initiate purchase to Blockchain.', true); - this._handleError(handler_id, 'Unable to initiate purchase to bc'); + await this._handleError(handler_id, 'Unable to initiate purchase to bc'); return Command.empty(); } @@ -84,11 +97,12 @@ class DvPurchaseInitiateCommand extends Command { purchase_id: purchaseId, permissioned_data_array_length, permissioned_data_original_length, + permissioned_data_root_hash, }; await this.commandExecutor.add({ name: 'dvPurchaseKeyDepositedCommand', - delay: 2 * 60 * 1000, // todo check why isn't it reading the default value + delay: 2 * 60 * 1000, retries: 3, data: commandData, }); @@ -122,6 +136,7 @@ class DvPurchaseInitiateCommand extends Command { } async _handleError(handler_id, errorMessage) { + this.logger.error(`Failed to initiate purchase: ${errorMessage}`); const handlerData = await this._getHandlerData(handler_id); await Models.data_trades.update({ diff --git a/modules/command/dv/dv-purchase-key-deposited-command.js b/modules/command/dv/dv-purchase-key-deposited-command.js index 99f6e081c..e3e5999aa 100644 --- a/modules/command/dv/dv-purchase-key-deposited-command.js +++ b/modules/command/dv/dv-purchase-key-deposited-command.js @@ -1,6 +1,7 @@ const Command = require('../command'); const Models = require('../../../models'); const Utilities = require('../../Utilities'); +const constants = require('../../constants'); /** * Handles data location response. @@ -9,6 +10,8 @@ class DvPurchaseKeyDepositedCommand extends Command { constructor(ctx) { super(ctx); this.remoteControl = ctx.remoteControl; + this.web3 = ctx.web3; + this.transport = ctx.transport; this.logger = ctx.logger; this.config = ctx.config; this.commandExecutor = ctx.commandExecutor; @@ -28,6 +31,7 @@ class DvPurchaseKeyDepositedCommand extends Command { purchase_id, permissioned_data_array_length, permissioned_data_original_length, + permissioned_data_root_hash, } = command.data; const events = await Models.events.findAll({ @@ -46,34 +50,65 @@ class DvPurchaseKeyDepositedCommand extends Command { if (event) { event.finished = true; await event.save({ fields: ['finished'] }); - this.logger.important(`Purchase ${purchase_id} verified. Decoding data from given key`); + this.logger.important(`Purchase ${purchase_id} confirmed by seller. Decoding data from submitted key.`); this.remoteControl.purchaseStatus('Purchase confirmed', 'Validating and storing data on your local node.'); const { key } = JSON.parse(event.data); - const decodedPermissionedData = this.permissionedDataService - .validateAndDecodePermissionedData( - encoded_data, key, permissioned_data_array_length, - permissioned_data_original_length, + + const decoded_data = this.permissionedDataService.decodePermissionedData( + encoded_data, + key, + ); + + const validationResult = this.permissionedDataService.validatePermissionedDataTree( + decoded_data, + permissioned_data_array_length, + ); + + const rootIsValid = this.permissionedDataService.validatePermissionedDataRoot( + decoded_data, + permissioned_data_root_hash, + ); + + if (validationResult.error || !rootIsValid) { + let errorMessage; + + if (validationResult.error) { + command.data.input_index_left = validationResult.inputIndexLeft; + command.data.output_index = validationResult.outputIndex; + command.data.error_type = constants.PURCHASE_ERROR_TYPE.NODE_ERROR; + errorMessage = 'Detected error in permissioned data merkle tree.'; + } else if (!rootIsValid) { + command.data.error_type = constants.PURCHASE_ERROR_TYPE.ROOT_ERROR; + errorMessage = 'Detected error in permissioned data decoded root hash.'; + } + + await this._handleError( + handler_id, + purchase_id, + errorMessage, ); - if (decodedPermissionedData.errorStatus) { - const commandData = { - encoded_data, - }; + + command.data.key = key; await this.commandExecutor.add({ name: 'dvPurchaseDisputeCommand', - data: commandData, + data: command.data, }); return Command.empty(); } + const reconstructedPermissionedData = this.permissionedDataService + .reconstructPermissionedData( + decoded_data, + permissioned_data_array_length, + permissioned_data_original_length, + ); + const handler = await Models.handler_ids.findOne({ where: { handler_id, }, }); - handler.status = 'COMPLETED'; - await handler.save({ fields: ['status'] }); - const { data_set_id, ot_object_id, @@ -82,9 +117,12 @@ class DvPurchaseKeyDepositedCommand extends Command { await this.permissionedDataService.updatePermissionedDataInDb( data_set_id, ot_object_id, - decodedPermissionedData.permissionedData, + reconstructedPermissionedData, ); + handler.status = 'COMPLETED'; + await handler.save({ fields: ['status'] }); + await Models.data_trades.update( { status: 'COMPLETED', @@ -105,6 +143,29 @@ class DvPurchaseKeyDepositedCommand extends Command { }); this.logger.important(`Purchase ${purchase_id} completed. Data stored successfully`); this.remoteControl.purchaseStatus('Purchase completed', 'You can preview the purchased data in My Purchases page.'); + + const purchaseCompletionObject = { + message: { + purchase_id, + data_set_id, + ot_object_id, + seller_node_id: this.config.identity, + seller_erc_id: Utilities.normalizeHex(this.config.erc725Identity), + price: this.config.default_data_price, + wallet: this.config.node_wallet, + }, + }; + + purchaseCompletionObject.messageSignature = + Utilities.generateRsvSignature( + purchaseCompletionObject.message, + this.web3, + this.config.node_private_key, + ); + + await this.transport.publish('kad-purchase-complete', purchaseCompletionObject); + this.logger.info('Published purchase confirmation on the network.'); + return Command.empty(); } } @@ -122,12 +183,13 @@ class DvPurchaseKeyDepositedCommand extends Command { async recover(command, err) { const { handler_id, purchase_id } = command.data; - await this._handleError(handler_id, purchase_id, `Failed to process dvPurchaseKeyDepositedCommand. Error: ${err}`); + await this._handleError(handler_id, purchase_id, err); return Command.empty(); } async _handleError(handler_id, purchase_id, errorMessage) { + this.logger.error(`Error occured in dvPurchaseKeyDepositedCommand. Reason given: ${errorMessage}`); await Models.data_trades.update({ status: 'FAILED', }, { diff --git a/modules/constants.js b/modules/constants.js index 4019e3178..b6d11df8b 100644 --- a/modules/constants.js +++ b/modules/constants.js @@ -100,6 +100,15 @@ exports.ANSWER_LITIGATION_COMMAND_RETRIES = 2; * Minimal number of blocks which are used for creating permissioned data merkle tree */ exports.NUMBER_OF_PERMISSIONED_DATA_FIRST_LEVEL_BLOCKS = 256; +/** + * + * @constant {object} PURCHASE_ERROR_TYPE - + * Types of errors supported for permissioned data purchase + */ +exports.PURCHASE_ERROR_TYPE = { + NODE_ERROR: 'node_error', + ROOT_ERROR: 'root_error', +}; /** * * @constant {integer} PUBLIC_KEY_VALIDITY_IN_MILLS - diff --git a/modules/controller/dc-controller.js b/modules/controller/dc-controller.js index 88e04388b..0fdcf8840 100644 --- a/modules/controller/dc-controller.js +++ b/modules/controller/dc-controller.js @@ -2,6 +2,8 @@ const utilities = require('../Utilities'); const Models = require('../../models'); const Utilities = require('../Utilities'); const constants = require('../constants'); +const { QueryTypes } = require('sequelize'); +const BN = require('bn.js'); /** * DC related API controller @@ -107,131 +109,104 @@ class DCController { } } - // async handlePrivateDataReadRequest(message) { - // const { - // data_set_id, dv_erc725_identity, ot_object_id, handler_id, nodeId, - // } = message; - // - // const privateDataPermissions = await Models.data_trades.findAll({ - // where: { - // data_set_id, - // ot_json_object_id: ot_object_id, - // buyer_node_id: nodeId, - // status: 'COMPLETED', - // }, - // }); - // if (!privateDataPermissions || privateDataPermissions.length === 0) { - // throw Error(`You don't have permission to view objectId: - // ${ot_object_id} from dataset: ${data_set_id}`); - // } - // - // const replayMessage = { - // wallet: this.config.node_wallet, - // handler_id, - // }; - // const promises = []; - // privateDataPermissions.forEach((privateDataPermisssion) => { - // promises.push(this.graphStorage.findDocumentsByImportIdAndOtObjectId( - // data_set_id, - // privateDataPermisssion.ot_json_object_id, - // )); - // }); - // const otObjects = await Promise.all(promises); - // replayMessage.ot_objects = otObjects; - // - // privateDataPermissions.forEach(async (privateDataPermisssion) => { - // await Models.data_sellers.create({ - // data_set_id, - // ot_json_object_id: privateDataPermisssion.ot_json_object_id, - // seller_node_id: nodeId.toLowerCase(), - // seller_erc_id: Utilities.normalizeHex(dv_erc725_identity), - // price: 0, - // }); - // }); - // - // const privateDataReadResponseObject = { - // message: replayMessage, - // messageSignature: Utilities.generateRsvSignature( - // JSON.stringify(replayMessage), - // this.web3, - // this.config.node_private_key, - // ), - // }; - // await this.transport.sendPrivateDataReadResponse( - // privateDataReadResponseObject, - // nodeId, - // ); - // } - // async handleNetworkPurchaseRequest(request) { - // const { - // data_set_id, dv_erc725_identity, handler_id, dv_node_id, ot_json_object_id, - // } = request; - // - // const permission = await Models.data_trades.findOne({ - // where: { - // buyer_node_id: dv_node_id, - // data_set_id, - // ot_json_object_id, - // status: 'COMPLETED', - // }, - // }); - // let message = ''; - // let status = ''; - // const sellingData = await Models.data_sellers.findOne({ - // where: { - // data_set_id, - // ot_json_object_id, - // seller_node_id: this.config.identity, - // }, - // }); - // - // if (permission) { - // message = 'Data already purchased!'; - // status = 'COMPLETED'; - // } else if (!sellingData) { - // status = 'FAILED'; - // message = 'I dont have requested data'; - // } else { - // await Models.data_trades.create({ - // data_set_id, - // ot_json_object_id, - // buyer_node_id: dv_node_id, - // buyer_erc_id: dv_erc725_identity, - // seller_node_id: this.config.identity, - // seller_erc_id: this.config.erc725Identity.toLowerCase(), - // price: sellingData.price, - // purchase_id: '', - // status: 'COMPLETED', - // }); - // message = 'Data purchase successfully finalized!'; - // status = 'COMPLETED'; - // } - // - // - // const response = { - // handler_id, - // status, - // wallet: this.config.node_wallet, - // message, - // price: sellingData.price, - // seller_node_id: this.config.identity, - // seller_erc_id: this.config.erc725Identity, - // }; - // - // const dataPurchaseResponseObject = { - // message: response, - // messageSignature: Utilities.generateRsvSignature( - // JSON.stringify(response), - // this.web3, - // this.config.node_private_key, - // ), - // }; - // - // await this.transport.sendDataPurchaseResponse( - // dataPurchaseResponseObject, - // dv_node_id, - // ); - // } + async updatePermissionedDataPrice(req, res) { + this.logger.api('POST: Set permissioned data price.'); + if (req.body == null + || req.body.data_set_id == null + || req.body.ot_object_ids == null) { + res.status(400); + res.send({ message: 'Params data_set_id and ot_object_ids are required.' }); + return; + } + + const promises = []; + req.body.ot_object_ids.forEach((ot_object) => { + promises.push(new Promise(async (accept, reject) => { + const condition = { + seller_erc_id: this.config.erc725Identity.toLowerCase(), + data_set_id: req.body.data_set_id.toLowerCase(), + ot_json_object_id: ot_object.id, + }; + + const data = await Models.data_sellers.findOne({ + where: condition, + }); + + if (data) { + await Models.data_sellers.update( + { price: ot_object.price_in_trac }, + { where: { id: data.id } }, + ); + accept(); + } else { + reject(); + } + })); + }); + await Promise.all(promises).then(() => { + res.status(200); + res.send({ status: 'COMPLETED' }); + }); + } + + async handlePermissionedDataReadRequest(message) { + const { + data_set_id, dv_erc725_identity, ot_object_id, handler_id, nodeId, + } = message; + + const privateDataPermissions = await Models.data_trades.findAll({ + where: { + data_set_id, + ot_json_object_id: ot_object_id, + buyer_node_id: nodeId, + status: 'COMPLETED', + }, + }); + if (!privateDataPermissions || privateDataPermissions.length === 0) { + throw Error(`You don't have permission to view objectId: + ${ot_object_id} from dataset: ${data_set_id}`); + } + + const replayMessage = { + wallet: this.config.node_wallet, + handler_id, + }; + const promises = []; + privateDataPermissions.forEach((privateDataPermisssion) => { + promises.push(this.graphStorage.findDocumentsByImportIdAndOtObjectId( + data_set_id, + privateDataPermisssion.ot_json_object_id, + )); + }); + const otObjects = await Promise.all(promises); + replayMessage.ot_objects = otObjects; + + const normalized_dv_erc725_identity = Utilities.normalizeHex(dv_erc725_identity); + + privateDataPermissions.forEach(async (privateDataPermisssion) => { + await Models.data_sellers.create({ + data_set_id, + ot_json_object_id: privateDataPermisssion.ot_json_object_id, + seller_node_id: nodeId.toLowerCase(), + seller_erc_id: normalized_dv_erc725_identity, + price: 0, + }); + }); + + const privateDataReadResponseObject = { + message: replayMessage, + messageSignature: Utilities.generateRsvSignature( + JSON.stringify(replayMessage), + this.web3, + this.config.node_private_key, + ), + }; + await this.transport.sendPermissionedDataReadResponse( + privateDataReadResponseObject, + nodeId, + ); + } + async handleNetworkPurchaseRequest(request) { const { data_set_id, dv_erc725_identity, handler_id, dv_node_id, ot_json_object_id, price, @@ -256,6 +231,96 @@ class DCController { }); } + async getPermissionedDataOwned(req, res) { + this.logger.api('GET: Permissioned Data Owned.'); + + const query = 'SELECT ds.data_set_id, ds.ot_json_object_id, ds.price, ( SELECT Count(*) FROM data_trades dt Where dt.seller_erc_id = ds.seller_erc_id and ds.data_set_id = dt.data_set_id and ds.ot_json_object_id = dt.ot_json_object_id ) as sales FROM data_sellers ds where ds.seller_erc_id = :seller_erc '; + const data = await Models.sequelize.query( + query, + { + replacements: { seller_erc: Utilities.normalizeHex(this.config.erc725Identity) }, + type: QueryTypes.SELECT, + }, + ); + + const result = []; + + if (data.length > 0) { + const owned_objects = {}; + const allDatasets = []; + /* + Creating a map of the following structure + owned_objects: { + dataset_0x456: { + ot_objects: [ot_object_0x789, ...] + ..., + }, + ... + } + */ + data.forEach((obj) => { + if (owned_objects[obj.data_set_id]) { + owned_objects[obj.data_set_id].ot_objects.push({ + id: obj.ot_json_object_id, + price: obj.price, + sales: obj.sales, + }); + owned_objects[obj.data_set_id].total_sales.iadd(new BN(obj.sales, 10)); + owned_objects[obj.data_set_id].total_price.iadd(new BN(obj.price, 10)); + } else { + allDatasets.push(obj.data_set_id); + owned_objects[obj.data_set_id] = {}; + owned_objects[obj.data_set_id].ot_objects = [{ + id: obj.ot_json_object_id, + price: obj.price, + sales: obj.sales, + }]; + owned_objects[obj.data_set_id].total_sales = new BN(obj.sales, 10); + owned_objects[obj.data_set_id].total_price = new BN(obj.price, 10); + } + }); + + const allMetadata = await this.importService.getMultipleDatasetMetadata(allDatasets); + + const dataInfos = await Models.data_info.findAll({ + where: { + data_set_id: { + [Models.sequelize.Op.in]: allDatasets, + }, + }, + }); + + allDatasets.forEach((datasetId) => { + const { datasetHeader } = allMetadata.find(metadata => metadata._key === datasetId); + const dataInfo = dataInfos.find(info => info.data_set_id === datasetId); + owned_objects[datasetId].metadata = { + datasetTitle: datasetHeader.datasetTitle, + datasetTags: datasetHeader.datasetTags, + datasetDescription: datasetHeader.datasetDescription, + timestamp: dataInfo.import_timestamp, + }; + }); + + for (const dataset in owned_objects) { + result.push({ + timestamp: (new Date(owned_objects[dataset].metadata.timestamp)).getTime(), + dataset: { + id: dataset, + name: owned_objects[dataset].metadata.datasetTitle, + description: owned_objects[dataset].metadata.datasetDescription || 'No description given', + tags: owned_objects[dataset].metadata.datasetTags, + }, + ot_objects: owned_objects[dataset].ot_objects, + total_sales: owned_objects[dataset].total_sales.toString(), + total_price: owned_objects[dataset].total_price.toString(), + }); + } + } + + res.status(200); + res.send(result); + } + async handleNetworkPriceRequest(request) { const { data_set_id, handler_id, dv_node_id, ot_json_object_id, diff --git a/modules/controller/dv-controller.js b/modules/controller/dv-controller.js index 4a36fc354..b2e05b7ac 100644 --- a/modules/controller/dv-controller.js +++ b/modules/controller/dv-controller.js @@ -3,6 +3,7 @@ const Models = require('../../models'); const Utilities = require('../Utilities'); const ImportUtilities = require('../ImportUtilities'); const constants = require('../constants'); +const { QueryTypes } = require('sequelize'); /** * Encapsulates DV related methods */ @@ -18,12 +19,20 @@ class DVController { this.config = ctx.config; this.web3 = ctx.web3; this.graphStorage = ctx.graphStorage; + this.importService = ctx.importService; this.mapping_standards_for_event = new Map(); this.mapping_standards_for_event.set('OT-JSON', 'ot-json'); this.mapping_standards_for_event.set('GS1-EPCIS', 'gs1'); this.mapping_standards_for_event.set('GRAPH', 'ot-json'); this.mapping_standards_for_event.set('WOT', 'wot'); + + this.trading_type_purchased = 'PURCHASED'; + this.trading_type_sold = 'SOLD'; + this.trading_type_all = 'ALL'; + this.trading_types = [ + this.trading_type_purchased, this.trading_type_sold, this.trading_type_all, + ]; } /** @@ -96,6 +105,64 @@ class DVController { response.send(responses); } + async getTradingData(req, res) { + this.logger.api('GET: Get trading data.'); + const requestedType = req.params.type; + if (!requestedType || !this.trading_types.includes(requestedType)) { + res.status(400); + res.send({ + message: 'Param type with values: PURCHASED, SOLD or ALL is required.', + }); + } + const normalizedIdentity = Utilities.normalizeHex(this.config.erc725Identity); + const whereCondition = {}; + if (requestedType === this.trading_type_purchased) { + whereCondition.buyer_erc_id = normalizedIdentity; + } else if (requestedType === this.trading_type_sold) { + whereCondition.seller_erc_id = normalizedIdentity; + } + + const tradingData = await Models.data_trades.findAll({ + where: whereCondition, + order: [ + ['timestamp', 'DESC'], + ], + }); + + const allDatasets = tradingData.map(element => element.data_set_id) + .filter((value, index, self) => self.indexOf(value) === index); + + const allMetadata = await this.importService.getMultipleDatasetMetadata(allDatasets); + + const returnArray = []; + tradingData.forEach((element) => { + const { datasetHeader } = + allMetadata.find(metadata => metadata._key === element.data_set_id); + const type = normalizedIdentity === element.buyer_erc_id ? 'PURCHASED' : 'SOLD'; + returnArray.push({ + data_set: { + id: element.data_set_id, + name: datasetHeader.datasetTitle, + description: datasetHeader.datasetDescription, + tags: datasetHeader.datasetTags, + }, + ot_json_object_id: element.ot_json_object_id, + buyer_erc_id: element.buyer_erc_id, + buyer_node_id: element.buyer_node_id, + seller_erc_id: element.seller_erc_id, + seller_node_id: element.seller_node_id, + price_in_trac: element.price_in_trac, + purchase_id: element.purchase_id, + timestamp: element.timestamp, + type, + status: element.status, + }); + }); + + res.status(200); + res.send(returnArray); + } + /** * Handles data read request * @param data_set_id - Dataset to be read @@ -164,127 +231,238 @@ class DVController { } } - // /** - // * Handles private data read request - // * @param data_set_id - Dataset that holdsprivate data - // * @param ot_object_id - Object id that holds private data - // * @param seller_node_id - Node id that holds private data - // * @param res - API result object - // * @returns null - // */ - // async handlePermissionedDataReadRequest(data_set_id, ot_object_id,seller_node_id,response) { - // const handler_data = { - // data_set_id, - // ot_object_id, - // seller_node_id, - // }; - // const inserted_object = await Models.handler_ids.create({ - // status: 'PENDING', - // data: JSON.stringify(handler_data), - // }); - // const handlerId = inserted_object.dataValues.handler_id; - // this.logger.info(`Read private data with id - // ${ot_object_id} with handler id ${handlerId} initiated.`); - // - // response.status(200); - // response.send({ - // handler_id: handlerId, - // }); - // - // this.commandExecutor.add({ - // name: 'dvPermissionedDataReadRequestCommand', - // delay: 0, - // data: { - // data_set_id, - // ot_object_id, - // seller_node_id, - // handlerId, - // }, - // transactional: false, - // }); - // } - - // _validatePermissionedData(data) { - // let validated = false; - // constants.PRIVATE_DATA_OBJECT_NAMES.forEach((private_data_array) => { - // if (data[private_data_array] && Array.isArray(data[private_data_array])) { - // data[private_data_array].forEach((private_object) => { - // if (private_object.isPrivate && private_object.data) { - // const calculatedPrivateHash = ImportUtilities - // .calculatePermissionedDataHash(private_object); - // validated = calculatedPrivateHash ===private_object.permissioned_data_hash; - // } - // }); - // } - // }); - // return validated; - // } - // - // async handlePermissionedDataReadResponse(message) { - // const { - // handler_id, ot_objects, - // } = message; - // const documentsToBeUpdated = []; - // ot_objects.forEach((otObject) => { - // otObject.relatedObjects.forEach((relatedObject) => { - // if (relatedObject.vertex.vertexType === 'Data') { - // if (this._validatePermissionedData(relatedObject.vertex.data)) { - // documentsToBeUpdated.push(relatedObject.vertex); - // } - // } - // }); - // }); - // const promises = []; - // documentsToBeUpdated.forEach((document) => { - // promises.push(this.graphStorage.updateDocument('ot_vertices', document)); - // }); - // await Promise.all(promises); - // - // const handlerData = await Models.handler_ids.findOne({ - // where: { - // handler_id, - // }, - // }); - // - // const { data_set_id, ot_object_id } = JSON.parse(handlerData.data); - // - // await Models.data_sellers.create({ - // data_set_id, - // ot_json_object_id: ot_object_id, - // seller_node_id: this.config.identity.toLowerCase(), - // seller_erc_id: Utilities.normalizeHex(this.config.erc725Identity), - // price: this.config.default_data_price, - // }); - // - // - // await Models.handler_ids.update({ - // status: 'COMPLETED', - // }, { where: { handler_id } }); - // } - - // async sendNetworkPurchase(dataSetId, erc725Identity, nodeId, otJsonObjectId, handlerId) { - // const message = { - // data_set_id: dataSetId, - // dv_erc725_identity: erc725Identity, - // handler_id: handlerId, - // ot_json_object_id: otJsonObjectId, - // wallet: this.config.node_wallet, - // }; - // const dataPurchaseRequestObject = { - // message, - // messageSignature: Utilities.generateRsvSignature( - // JSON.stringify(message), - // this.web3, - // this.config.node_private_key, - // ), - // }; - // await this.transport.sendDataPurchaseRequest( - // dataPurchaseRequestObject, - // nodeId, - // ); - // } + /** + * Handles permissioned data read request + * @param request + * @param response + * @returns null + */ + async handlePermissionedDataReadRequest(request, response) { + this.logger.api('Private data network read request received.'); + + if (!request.body || !request.body.seller_node_id + || !request.body.data_set_id + || !request.body.ot_object_id) { + request.status(400); + request.send({ message: 'Params data_set_id,ot_object_id and seller_node_id are required.' }); + } + const { data_set_id, ot_object_id, seller_node_id } = request.body; + const handler_data = { + data_set_id, + ot_object_id, + seller_node_id, + }; + const inserted_object = await Models.handler_ids.create({ + status: 'PENDING', + data: JSON.stringify(handler_data), + }); + const { handler_id } = inserted_object.dataValues; + this.logger.info(`Read private data with id ${ot_object_id} with handler id ${handler_id} initiated.`); + + response.status(200); + response.send({ + handler_id, + }); + + this.commandExecutor.add({ + name: 'dvPermissionedDataReadRequestCommand', + delay: 0, + data: { + data_set_id, + ot_object_id, + seller_node_id, + handler_id, + }, + transactional: false, + }); + } + + async handlePermissionedDataReadResponse(message) { + const { + handler_id, ot_objects, + } = message; + const documentsToBeUpdated = []; + ot_objects.forEach((otObject) => { + otObject.relatedObjects.forEach((relatedObject) => { + if (relatedObject.vertex.vertexType === 'Data') { + const permissionedDataHash = ImportUtilities + .calculatePermissionedDataHash(relatedObject.vertex.data.permissioned_data); + if (permissionedDataHash !== relatedObject.vertex.data.permissioned_data.permissioned_data_hash) { throw new Error(`Calculated permissioned data hash ${permissionedDataHash} differs from DC permissioned data hash ${relatedObject.vertex.data.permissioned_data.permissioned_data_hash}`); } + documentsToBeUpdated.push(relatedObject.vertex); + } + }); + }); + const promises = []; + documentsToBeUpdated.forEach((document) => { + promises.push(this.graphStorage.updateDocument('ot_vertices', document)); + }); + await Promise.all(promises); + + const handlerData = await Models.handler_ids.findOne({ + where: { + handler_id, + }, + }); + + const { data_set_id, ot_object_id } = JSON.parse(handlerData.data); + + await Models.data_sellers.create({ + data_set_id, + ot_json_object_id: ot_object_id, + seller_node_id: this.config.identity.toLowerCase(), + seller_erc_id: Utilities.normalizeHex(this.config.erc725Identity), + price: this.config.default_data_price, + }); + + + await Models.handler_ids.update({ + status: 'COMPLETED', + }, { where: { handler_id } }); + } + + + async getPermissionedDataAvailable(req, res) { + this.logger.api('GET: Permissioned data Available for purchase.'); + + const query = 'SELECT * FROM data_sellers DS WHERE NOT EXISTS(SELECT * FROM data_sellers MY WHERE MY.seller_erc_id = :seller_erc AND MY.data_set_id = DS.data_set_id AND MY.ot_json_object_id = DS.ot_json_object_id)'; + const data = await Models.sequelize.query( + query, + { + replacements: { seller_erc: Utilities.normalizeHex(this.config.erc725Identity) }, + type: QueryTypes.SELECT, + }, + ); + + const result = []; + + if (data.length > 0) { + const not_owned_objects = {}; + const allDatasets = []; + /* + Creating a map of the following structure + not_owned_objects: { + dataset_0x456: { + seller_0x123: [ot_object_0x789, ...] + ..., + }, + ... + } + */ + data.forEach((obj) => { + if (not_owned_objects[obj.data_set_id]) { + if (not_owned_objects[obj.data_set_id][obj.seller_node_id]) { + not_owned_objects[obj.data_set_id][obj.seller_node_id] + .ot_json_object_id.push(obj.ot_json_object_id); + } else { + not_owned_objects[obj.data_set_id][obj.seller_node_id] = {}; + + not_owned_objects[obj.data_set_id][obj.seller_node_id] + .ot_json_object_id = [obj.ot_json_object_id]; + not_owned_objects[obj.data_set_id][obj.seller_node_id] + .seller_erc_id = obj.seller_erc_id; + } + } else { + allDatasets.push(obj.data_set_id); + + not_owned_objects[obj.data_set_id] = {}; + + not_owned_objects[obj.data_set_id][obj.seller_node_id] = {}; + + not_owned_objects[obj.data_set_id][obj.seller_node_id] + .ot_json_object_id = [obj.ot_json_object_id]; + not_owned_objects[obj.data_set_id][obj.seller_node_id] + .seller_erc_id = obj.seller_erc_id; + } + }); + + const allMetadata = await this.importService.getMultipleDatasetMetadata(allDatasets); + + const dataInfos = await Models.data_info.findAll({ + where: { + data_set_id: { + [Models.sequelize.Op.in]: allDatasets, + }, + }, + }); + + allDatasets.forEach((datasetId) => { + const { datasetHeader } = allMetadata.find(metadata => metadata._key === datasetId); + const dataInfo = dataInfos.find(info => info.data_set_id === datasetId); + not_owned_objects[datasetId].metadata = { + datasetTitle: datasetHeader.datasetTitle, + datasetTags: datasetHeader.datasetTags, + datasetDescription: datasetHeader.datasetDescription, + timestamp: dataInfo.import_timestamp, + creator_identity: ImportUtilities.getDataCreator(datasetHeader), + creator_wallet: dataInfo.data_provider_wallet, + }; + }); + + for (const dataset in not_owned_objects) { + for (const data_seller in not_owned_objects[dataset]) { + if (data_seller !== 'metadata') { + result.push({ + seller_node_id: data_seller, + timestamp: (new Date(not_owned_objects[dataset].metadata.timestamp)) + .getTime(), + dataset: { + id: dataset, + name: not_owned_objects[dataset].metadata.datasetTitle, + description: not_owned_objects[dataset].metadata.datasetDescription, + tags: not_owned_objects[dataset].metadata.datasetTags, + creator_wallet: not_owned_objects[dataset].metadata.creator_wallet, + creator_identity: + not_owned_objects[dataset].metadata.creator_identity, + }, + ot_objects: not_owned_objects[dataset][data_seller].ot_json_object_id, + seller_erc_id: not_owned_objects[dataset][data_seller].seller_erc_id, + }); + } + } + } + } + + res.status(200); + res.send(result); + } + + async getPermissionedDataPrice(req, res) { + this.logger.api('POST: Get permissioned data price.'); + if (req.body == null + || req.body.data_set_id == null + || req.body.seller_node_id == null + || req.body.ot_object_id == null) { + res.status(400); + res.send({ message: 'Params data_set_id, seller_node_id and ot_json_object_id are required.' }); + } + + const { + data_set_id, seller_node_id, ot_object_id, + } = req.body; + const inserted_object = await Models.handler_ids.create({ + data: JSON.stringify({ + data_set_id, seller_node_id, ot_object_id, + }), + status: 'PENDING', + }); + + const handlerId = inserted_object.dataValues.handler_id; + + await this.sendPermissionedDataPriceRequest( + data_set_id, + seller_node_id, + ot_object_id, + handlerId, + ); + + res.status(200); + res.send({ + handler_id: handlerId, + }); + } async sendNetworkPurchase(request, response) { + this.logger.api('POST: Permissioned data purchase request.'); if (request.body == null || request.body.data_set_id == null || request.body.seller_node_id == null @@ -342,41 +520,6 @@ class DVController { nodeId, ); } - // async handleNetworkPurchaseResponse(response) { - // const { - // handler_id, status, message, seller_node_id, seller_erc_id, price, - // } = response; - // - // const handlerData = await Models.handler_ids.findOne({ - // where: { - // handler_id, - // }, - // }); - // - // const { data_set_id, ot_object_id } = JSON.parse(handlerData.data); - // - // await Models.data_trades.create({ - // data_set_id, - // ot_json_object_id: ot_object_id, - // buyer_node_id: this.config.identity, - // buyer_erc_id: this.config.erc725Identity.toLowerCase(), - // seller_node_id, - // seller_erc_id, - // price, - // purchase_id: '', - // status, - // }); - // - // await Models.handler_ids.update({ - // data: JSON.stringify({ message }), - // status, - // }, { - // where: { - // handler_id, - // }, - // }); - // } - async handleNetworkPurchaseResponse(response) { const { @@ -650,6 +793,89 @@ class DVController { }); }); } + + /** + * Handle new purchase on the blockchain and add the node that bought data as a new + * data seller + * @param purchase_id + * @param seller_erc_id + * @param seller_node_id + * @param data_set_id + * @param ot_object_id + * @param price + * @returns {Promise} + */ + async handleNewDataSeller( + purchase_id, seller_erc_id, seller_node_id, + data_set_id, ot_object_id, price, + ) { + /* + * [x] Check that I have the dataset + * [x] Check that I don't have the ot-object + * [x] Check that the seller has a purchase on blockchain + * [x] Check that the ot-object exists in the dataset + * [x] Check that the permissioned data hash matches the hash in the ot-object + * */ + + const dataInfo = await Models.data_info.findAll({ where: { data_set_id } }); + if (!dataInfo || !Array.isArray(dataInfo) || !dataInfo.length > 0) { + this.logger.info(`Dataset ${data_set_id} not imported on node, skipping adding data seller.`); + return; + } + + const myDataPrice = await Models.data_sellers.findAll({ + where: { + data_set_id, + ot_json_object_id: ot_object_id, + seller_erc_id: Utilities.normalizeHex(this.config.erc725Identity), + }, + }); + + if (myDataPrice && Array.isArray(myDataPrice) && myDataPrice.length > 0) { + this.logger.info(`I already have permissioned data of object ${ot_object_id}` + + ` from dataset ${data_set_id}`); + return; + } + + const purchase = await this.blockchain.getPurchase(purchase_id); + const { + seller, + buyer, + originalDataRootHash, + } = purchase; + + if (Utilities.normalizeHex(buyer) !== Utilities.normalizeHex(seller_erc_id)) { + this.logger.warn('New data seller\'s ERC-725 identity does not match' + + ` the purchase buyer identity ${Utilities.normalizeHex(buyer)}`); + return; + } + + const otObject = await this.importService.getOtObjectById(data_set_id, ot_object_id); + if (!otObject || !otObject.properties || !otObject.properties.permissioned_data) { + this.logger.info(`Object ${ot_object_id} not found in dataset ${data_set_id} ` + + 'or does not contain permissioned data.'); + return; + } + + const permissionedDataHash = otObject.properties.permissioned_data.permissioned_data_hash; + if (Utilities.normalizeHex(permissionedDataHash) !== + Utilities.normalizeHex(originalDataRootHash)) { + this.logger.info('Purchase permissioned data root hash does not match ' + + 'the permissioned data root hash from dataset.'); + return; + } + + await Models.data_sellers.create({ + data_set_id, + ot_json_object_id: ot_object_id, + seller_node_id, + seller_erc_id, + price, + }); + + this.logger.notify(`Saved ${seller_node_id} as new seller for permissioned data ` + + `of object ${ot_object_id} from dataset ${data_set_id}`); + } } module.exports = DVController; diff --git a/modules/network/kademlia/kademlia.js b/modules/network/kademlia/kademlia.js index 05072a49c..adf6488bc 100644 --- a/modules/network/kademlia/kademlia.js +++ b/modules/network/kademlia/kademlia.js @@ -34,6 +34,7 @@ const directMessageRequests = [ { methodName: 'dataReadRequest', routeName: 'kad-data-read-request' }, { methodName: 'sendDataReadResponse', routeName: 'kad-data-read-response' }, { methodName: 'sendPermissionedDataReadRequest', routeName: 'kad-permissioned-data-read-request' }, + { methodName: 'sendPermissionedDataReadResponse', routeName: 'kad-permissioned-data-read-response' }, { methodName: 'sendDataPurchaseRequest', routeName: 'kad-data-purchase-request' }, { methodName: 'sendDataPurchaseResponse', routeName: 'kad-data-purchase-response' }, { methodName: 'sendPermissionedDataPriceRequest', routeName: 'kad-data-price-request' }, @@ -175,7 +176,7 @@ class Kademlia { 'kad-permissioned-data-read-response', 'kad-permissioned-data-read-request', 'kad-send-encrypted-key', 'kad-encrypted-key-process-result', 'kad-replication-request', 'kad-replacement-replication-request', 'kad-replacement-replication-finished', - 'kad-public-key-request', + 'kad-public-key-request', 'kad-purchase-complete', ], difficulty: this.config.network.solutionDifficulty, })); @@ -466,6 +467,11 @@ class Kademlia { this.emitter.emit('kad-data-location-request', message); }); + this.node.quasar.quasarSubscribe('kad-purchase-complete', (message, err) => { + this.log.info('New purchase completed on the network'); + this.emitter.emit('kad-purchase-complete', message); + }); + // sync this.node.use('kad-replication-request', (request, response, next) => { this.log.debug('kad-replication-request received'); @@ -555,6 +561,34 @@ class Kademlia { response.send([]); }); + // async + this.node.use('kad-data-purchase-request', (request, response, next) => { + this.log.debug('kad-data-purchase-request received'); + this.emitter.emit('kad-data-purchase-request', request); + response.send([]); + }); + + // async + this.node.use('kad-data-purchase-response', (request, response, next) => { + this.log.debug('kad-data-purchase-response received'); + this.emitter.emit('kad-data-purchase-response', request); + response.send([]); + }); + + // async + this.node.use('kad-data-price-request', (request, response, next) => { + this.log.debug('kad-data-price-request received'); + this.emitter.emit('kad-data-price-request', request); + response.send([]); + }); + + // async + this.node.use('kad-data-price-response', (request, response, next) => { + this.log.debug('kad-data-price-response received'); + this.emitter.emit('kad-data-price-response', request); + response.send([]); + }); + // async this.node.use('kad-send-encrypted-key', (request, response, next) => { this.log.debug('kad-send-encrypted-key received'); @@ -653,21 +687,6 @@ class Kademlia { node.getNearestNeighbour = () => [...node.router.getClosestContactsToKey(this.identity).entries()].shift(); - - node.sendUnpackedMessage = async (message, contactId, method) => { - const { contact, header } = await node.getContact(contactId); - return new Promise((resolve, reject) => { - node.send(method, { message, header }, contact, (err, res) => { - if (err) { - reject(err); - } else { - resolve(res); - } - }); - }); - }; - - directMessageRequests.forEach((element) => { node[element.methodName] = async (message, contactId) => node.sendDirectMessage(message, contactId, element.routeName); diff --git a/modules/service/permissioned-data-service.js b/modules/service/permissioned-data-service.js index 13b1c09f0..91d8b07ac 100644 --- a/modules/service/permissioned-data-service.js +++ b/modules/service/permissioned-data-service.js @@ -6,7 +6,6 @@ const crypto = require('crypto'); const Encryption = require('../RSAEncryption'); const abi = require('ethereumjs-abi'); const ImportUtilities = require('../ImportUtilities'); -const kadence = require('@deadcanaries/kadence'); class PermissionedDataService { constructor(ctx) { @@ -139,44 +138,68 @@ class PermissionedDataService { await Promise.all(promises); } - encodePermissionedData(permissionedObject) { - const merkleTree = ImportUtilities - .calculatePermissionedDataMerkleTree(permissionedObject, 'purchase'); + _encodePermissionedDataMerkleTree(merkleTree) { const rawKey = crypto.randomBytes(32); - const key = Utilities.normalizeHex(Buffer.from(`${rawKey}`, 'utf8').toString('hex').padStart(64, '0')); + const key = Utilities.normalizeHex(rawKey.toString('hex')); const encodedArray = []; + let index = 0; - merkleTree.levels.forEach((level) => { - for (let i = 0; i < level.length; i += 1) { - const leaf = level[i]; - const keyHash = abi.soliditySHA3( + for (let levelIndex = 1; levelIndex < merkleTree.levels.length; levelIndex += 1) { + const level = merkleTree.levels[levelIndex]; + for (let leafIndex = 0; leafIndex < level.length; leafIndex += 1, index += 1) { + const leaf = level[leafIndex]; + let keyHash = abi.soliditySHA3( ['bytes32', 'uint256'], [key, index], ).toString('hex'); + encodedArray.push(Encryption.xor(leaf, keyHash)); - index += 1; + + if (leafIndex === level.length - 1 && level.length % 2 === 1) { + index += 1; + keyHash = abi.soliditySHA3( + ['bytes32', 'uint256'], + [key, index], + ).toString('hex'); + encodedArray.push(Encryption.xor(leaf, keyHash)); + } } - }); - const encodedMerkleTree = new MerkleTree(encodedArray, 'purchase', 'sha3'); + } + const encodedMerkleTree = new MerkleTree(encodedArray, 'purchase', 'soliditySha3'); const encodedDataRootHash = encodedMerkleTree.getRoot(); - const sorted_data = Utilities.sortedStringify(permissionedObject.data, true); - const data = Buffer.from(sorted_data); return { - permissioned_data_original_length: data.length, permissioned_data_array_length: merkleTree.levels[0].length, key, encoded_data: encodedArray, - permissioned_data_root_hash: - Utilities.normalizeHex(permissionedObject.permissioned_data_hash), + permissioned_data_root_hash: Utilities.normalizeHex(merkleTree.getRoot()), encoded_data_root_hash: Utilities.normalizeHex(encodedDataRootHash), }; } - static validateAndDecodePermissionedData( - permissionedDataArray, key, - permissionedDataArrayLength, - permissionedDataOriginalLength, - ) { + encodePermissionedData(permissionedObject) { + const merkleTree = ImportUtilities + .calculatePermissionedDataMerkleTree(permissionedObject.properties.permissioned_data, 'purchase'); + + const result = this._encodePermissionedDataMerkleTree(merkleTree); + + const sorted_data = Utilities.sortedStringify( + permissionedObject.properties.permissioned_data.data, + true, + ); + + const data = Buffer.from(sorted_data); + result.permissioned_data_original_length = data.length; + + return result; + } + + /** + * Decodes the array of data with the given key + * @param permissionedDataArray - Array of elements encoded + * @param key - String key in hex form + * @returns {[]} - Decoded data + */ + decodePermissionedData(permissionedDataArray, key) { const decodedDataArray = []; permissionedDataArray.forEach((element, index) => { const keyHash = abi.soliditySHA3( @@ -186,25 +209,56 @@ class PermissionedDataService { decodedDataArray.push(Encryption.xor(element, keyHash)); }); - const originalDataArray = decodedDataArray.slice(0, permissionedDataArrayLength); - - // todo add validation - // const originalDataMarkleTree = new MerkleTree(originalDataArray, 'purchase', 'sha3'); - // var index = 0; - // originalDataMarkleTree.levels.forEach((level) => { - // level.forEach((leaf) => { - // if (leaf !== decodedDataArray[index]){ - // //found non matching index - // return { - // : {}, - // errorStatus: 'VALIDATION_FAILED', - // }; - // } - // index += 1; - // }); - // }); - - // recreate original object + return decodedDataArray; + } + + validatePermissionedDataTree(decodedMerkleTreeArray, firstLevelLength) { + const baseLevel = decodedMerkleTreeArray.slice(0, firstLevelLength); + const calculatedMerkleTree = new MerkleTree(baseLevel, 'purchase', 'soliditySha3'); + + let decodedIndex = 0; + let previousLevelStart = 0; + + for (let levelIndex = 1; levelIndex < calculatedMerkleTree.levels.length; levelIndex += 1) { + const level = calculatedMerkleTree.levels[levelIndex]; + + for (let leafIndex = 0; leafIndex < level.length; leafIndex += 1, decodedIndex += 1) { + if (level[leafIndex] !== decodedMerkleTreeArray[decodedIndex]) { + return { + error: true, + inputIndexLeft: (leafIndex * 2) + previousLevelStart, + outputIndex: decodedIndex, + }; + } + } + + if (level.length % 2 === 1) { + decodedIndex += 1; + } + + if (levelIndex > 1) { + const previousLevel = calculatedMerkleTree.levels[levelIndex - 1]; + previousLevelStart += previousLevel.length; + if (previousLevel.length % 2 === 1) { + previousLevelStart += 1; + } + } + } + + return {}; + } + + validatePermissionedDataRoot(decodedMerkleTreeArray, permissionedDataRootHash) { + return Utilities.normalizeHex(permissionedDataRootHash) === + Utilities.normalizeHex(decodedMerkleTreeArray[decodedMerkleTreeArray.length - 1]); + } + + reconstructPermissionedData( + decodedMerkleTreeArray, + firstLevelLength, + permissionedDataOriginalLength, + ) { + const originalDataArray = decodedMerkleTreeArray.slice(0, firstLevelLength); const first_level_blocks = constants.NUMBER_OF_PERMISSIONED_DATA_FIRST_LEVEL_BLOCKS; const default_block_size = constants.DEFAULT_CHALLENGE_BLOCK_SIZE_BYTES; @@ -220,8 +274,38 @@ class PermissionedDataService { originalDataString += block.toString(); } + return JSON.parse(originalDataString); + } + + prepareNodeDisputeData(encodedData, inputIndexLeft, outputIndex) { + const encodedMerkleTree = new MerkleTree(encodedData, 'purchase', 'soliditySha3'); + + const encodedInputLeft = encodedData[inputIndexLeft]; + const encodedOutput = encodedData[outputIndex]; + + const proofOfEncodedInputLeft = encodedMerkleTree.createProof(inputIndexLeft); + const proofOfEncodedOutput = encodedMerkleTree.createProof(outputIndex); + + return { + encodedInputLeft, + encodedOutput, + proofOfEncodedInputLeft, + proofOfEncodedOutput, + }; + } + + prepareRootDisputeData(encodedData) { + const encodedMerkleTree = new MerkleTree(encodedData, 'purchase', 'soliditySha3'); + + const rootHashIndex = encodedMerkleTree.levels[0].length - 1; + const encodedRootHash = encodedData[rootHashIndex]; + + const proofOfEncodedRootHash = encodedMerkleTree.createProof(rootHashIndex); + return { - permissionedData: JSON.parse(originalDataString), + rootHashIndex, + encodedRootHash, + proofOfEncodedRootHash, }; } @@ -232,8 +316,8 @@ class PermissionedDataService { otObjectId, ); const documentsToBeUpdated = []; - const calculatedPermissionedDataHash = this - .calculatePermissionedDataHash({ data: permissionedData }); + const calculatedPermissionedDataHash = + ImportUtilities.calculatePermissionedDataHash({ data: permissionedData }); otObject.relatedObjects.forEach((relatedObject) => { if (relatedObject.vertex.vertexType === 'Data') { const vertexData = relatedObject.vertex.data; diff --git a/modules/service/rest-api-v2.js b/modules/service/rest-api-v2.js index f084c0eaa..e363d9824 100644 --- a/modules/service/rest-api-v2.js +++ b/modules/service/rest-api-v2.js @@ -1,10 +1,7 @@ const path = require('path'); -const { QueryTypes } = require('sequelize'); const fs = require('fs'); -const BN = require('bn.js'); const pjson = require('../../package.json'); const RestAPIValidator = require('../validator/rest-api-validator'); -const ImportUtilities = require('../ImportUtilities'); const Utilities = require('../Utilities'); const Models = require('../../models'); @@ -32,12 +29,6 @@ class RestAPIServiceV2 { this.version_id = 'v2.0'; this.stanards = ['OT-JSON', 'GS1-EPCIS', 'GRAPH', 'WOT']; - this.trading_type_purchased = 'PURCHASED'; - this.trading_type_sold = 'SOLD'; - this.trading_type_all = 'ALL'; - this.trading_types = [ - this.trading_type_purchased, this.trading_type_sold, this.trading_type_all, - ]; this.graphStorage = ctx.graphStorage; this.mapping_standards_for_event = new Map(); this.mapping_standards_for_event.set('ot-json', 'ot-json'); @@ -125,25 +116,14 @@ class RestAPIServiceV2 { server.get(`/api/${this.version_id}/network/read/result/:handler_id`, async (req, res) => { await this._checkForHandlerStatus(req, res); }); - // - // server.post(`/api/${this.version_id}/network/permissioned_data/read`,async(req, res) => { - // await this._privateDataReadNetwork(req, res); - // }); - - // server.get(`/api/${this.version_id}/network/permissioned_data/read/resu - // lt/:handler_id`, async (req, res) => { - // await this._checkForHandlerStatus(req, res); - // }); - - // server.post(`/api/${this.version_id}/network/ - // permissioned_data/purchase`, async (req, res) => { - // await this.dvController.sendNetworkPurchase(req, res); - // }); - - // server.get(`/api/${this.version_id}/network/ - // permissioned_data/purchase/result/:handler_id`, async (req, res) => { - // await this._checkForHandlerStatus(req, res); - // }); + + server.post(`/api/${this.version_id}/network/permissioned_data/read`, async (req, res) => { + await this.dvController.handlePermissionedDataReadRequest(req, res); + }); + + server.get(`/api/${this.version_id}/network/permissioned_data/read/result/:handler_id`, async (req, res) => { + await this._checkForHandlerStatus(req, res); + }); server.post(`/api/${this.version_id}/permissioned_data/whitelist_viewer`, async (req, res) => { await this.dhController.whitelistViewer(req, res); @@ -161,35 +141,39 @@ class RestAPIServiceV2 { await this._getChallenges(req, res); }); - // server.get(`/api/${this.version_id}/permissioned_data/available`, async (req, res) => { - // await this._getPermissionedDataAvailable(req, res); - // }); - // - // - // server.get(`/api/${this.version_id}/permissioned_data/owned`, async (req, res) => { - // await this._getPermissionedDataOwned(req, res); - // }); - // - // server.post(`/api/${this.version_id}/network/ - // permissioned_data/get_price`, async (req, res) => { - // await this._getPermissionedDataPrice(req, res); - // }); - // - // server.post(`/api/${this.version_id}/ - // permissioned_data/update_price`, async (req, res) => { - // await this._updatePermissionedDataPrice(req, res); - // }); - // - // server.get(`/api/${this.version_id}/network/ - // permissioned_data/get_price/result/:handler_id`, async (req, res) => { - // await this._checkForHandlerStatus(req, res); - // }); - // - // server.get(`/api/${this.version_id}/ - // permissioned_data/trading_info/:type`, async (req, res) => { - // await this._getTradingData(req, res); - // }); + if (process.env.NODE_ENV !== 'mainnet') { + server.post(`/api/${this.version_id}/network/permissioned_data/purchase`, async (req, res) => { + await this.dvController.sendNetworkPurchase(req, res); + }); + + server.get(`/api/${this.version_id}/network/permissioned_data/purchase/result/:handler_id`, async (req, res) => { + await this._checkForHandlerStatus(req, res); + }); + server.get(`/api/${this.version_id}/permissioned_data/available`, async (req, res) => { + await this.dvController.getPermissionedDataAvailable(req, res); + }); + + + server.get(`/api/${this.version_id}/permissioned_data/owned`, async (req, res) => { + await this.dcController.getPermissionedDataOwned(req, res); + }); + + server.post(`/api/${this.version_id}/network/permissioned_data/get_price`, async (req, res) => { + await this.dvController.getPermissionedDataPrice(req, res); + }); + + server.post(`/api/${this.version_id}/permissioned_data/update_price`, async (req, res) => { + await this.dcController.updatePermissionedDataPrice(req, res); + }); + server.get(`/api/${this.version_id}/network/permissioned_data/get_price/result/:handler_id`, async (req, res) => { + await this._checkForHandlerStatus(req, res); + }); + + server.get(`/api/${this.version_id}/permissioned_data/trading_info/:type`, async (req, res) => { + await this.dvController.getTradingData(req, res); + }); + } /** Network related routes */ server.get(`/api/${this.version_id}/network/get-contact/:node_id`, async (req, res) => { @@ -498,21 +482,6 @@ class RestAPIServiceV2 { await this.dvController.handleDataReadRequest(data_set_id, reply_id, res); } - // async _privateDataReadNetwork(req, res) { - // this.logger.api('Private data network read request received.'); - // - // if (!req.body || !req.body.seller_node_id - // || !req.body.data_set_id - // || !req.body.ot_object_id) { - // res.status(400); - // res.send({ message: 'Params data_set_id, - // ot_object_id and seller_node_id are required.' }); - // } - // const { data_set_id, ot_object_id, seller_node_id } = req.body; - // await this.dvController - // .handlePermissionedDataReadRequest(data_set_id, ot_object_id, seller_node_id, res); - // } - async _checkForReplicationHandlerStatus(req, res) { const handler_object = await Models.handler_ids.findOne({ where: { @@ -807,361 +776,6 @@ class RestAPIServiceV2 { }); } } - - - // async _networkPurchase(req, res) { - // this.logger.api('POST: Network purchase request received.'); - // - // if (req.body == null - // || req.body.data_set_id == null - // || req.body.seller_node_id == null - // || req.body.ot_object_id == null) { - // res.status(400); - // res.send({ message: ' - // Params data_set_id, seller_node_id and ot_object_id are required.' }); - // return; - // } - // const { - // data_set_id, seller_node_id, ot_object_id, - // } = req.body; - // const inserted_object = await Models.handler_ids.create({ - // data: JSON.stringify({ - // data_set_id, seller_node_id, ot_object_id, - // }), - // status: 'PENDING', - // }); - // const handlerId = inserted_object.dataValues.handler_id; - // res.status(200); - // res.send({ - // handler_id: handlerId, - // }); - // - // await this.dvController.sendNetworkPurchase( - // data_set_id, - // this.config.erc725Identity, - // seller_node_id, - // ot_object_id, - // handlerId, - // ); - // } - - async _getPermissionedDataAvailable(req, res) { - this.logger.api('GET: Permissioned data Available for purchase.'); - - const query = 'SELECT * FROM data_sellers DS WHERE NOT EXISTS(SELECT * FROM data_sellers MY WHERE MY.seller_erc_id = :seller_erc AND MY.data_set_id = DS.data_set_id AND MY.ot_json_object_id = DS.ot_json_object_id)'; - const data = await Models.sequelize.query( - query, - { - replacements: { seller_erc: Utilities.normalizeHex(this.config.erc725Identity) }, - type: QueryTypes.SELECT, - }, - ); - - const result = []; - - if (data.length > 0) { - const not_owned_objects = {}; - const allDatasets = []; - /* - Creating a map of the following structure - not_owned_objects: { - dataset_0x456: { - seller_0x123: [ot_object_0x789, ...] - ..., - }, - ... - } - */ - data.forEach((obj) => { - if (not_owned_objects[obj.data_set_id]) { - if (not_owned_objects[obj.data_set_id][obj.seller_node_id]) { - not_owned_objects[obj.data_set_id][obj.seller_node_id].ot_json_object_id - .push(obj.ot_json_object_id); - } else { - not_owned_objects[obj.data_set_id][obj.seller_node_id].ot_json_object_id - = [obj.ot_json_object_id]; - not_owned_objects[obj.data_set_id][obj.seller_node_id].seller_erc_id - = obj.seller_erc_id; - } - } else { - allDatasets.push(obj.data_set_id); - not_owned_objects[obj.data_set_id] = {}; - not_owned_objects[obj.data_set_id][obj.seller_node_id] = {}; - not_owned_objects[obj.data_set_id][obj.seller_node_id].ot_json_object_id - = [obj.ot_json_object_id]; - not_owned_objects[obj.data_set_id][obj.seller_node_id].seller_erc_id - = obj.seller_erc_id; - } - }); - - const allMetadata = await this.importService.getMultipleDatasetMetadata(allDatasets); - - const dataInfos = await Models.data_info.findAll({ - where: { - data_set_id: { - [Models.sequelize.Op.in]: allDatasets, - }, - }, - }); - - allDatasets.forEach((datasetId) => { - const { datasetHeader } = allMetadata.find(metadata => metadata._key === datasetId); - const dataInfo = dataInfos.find(info => info.data_set_id === datasetId); - not_owned_objects[datasetId].metadata = { - datasetTitle: datasetHeader.datasetTitle, - datasetTags: datasetHeader.datasetTags, - datasetDescription: datasetHeader.datasetDescription, - timestamp: dataInfo.import_timestamp, - }; - }); - - for (const dataset in not_owned_objects) { - for (const data_seller in not_owned_objects[dataset]) { - if (data_seller !== 'metadata') { - result.push({ - seller_node_id: data_seller, - timestamp: (new Date(not_owned_objects[dataset].metadata.timestamp)) - .getTime(), - dataset: { - id: dataset, - name: not_owned_objects[dataset].metadata.datasetTitle, - description: not_owned_objects[dataset].metadata.datasetDescription, - tags: not_owned_objects[dataset].metadata.datasetTags, - }, - ot_objects: not_owned_objects[dataset][data_seller].ot_json_object_id, - seller_erc_id: not_owned_objects[dataset][data_seller].seller_erc_id, - }); - } - } - } - } - - res.status(200); - res.send(result); - } - - async _getPermissionedDataOwned(req, res) { - this.logger.api('GET: Permissioned Data Owned.'); - - const query = 'SELECT ds.data_set_id, ds.ot_json_object_id, ds.price, ( SELECT Count(*) FROM data_trades dt Where dt.seller_erc_id = ds.seller_erc_id and ds.data_set_id = dt.data_set_id and ds.ot_json_object_id = dt.ot_json_object_id ) as sales FROM data_sellers ds where ds.seller_erc_id = :seller_erc '; - const data = await Models.sequelize.query( - query, - { - replacements: { seller_erc: Utilities.normalizeHex(this.config.erc725Identity) }, - type: QueryTypes.SELECT, - }, - ); - - const result = []; - - if (data.length > 0) { - const owned_objects = {}; - const allDatasets = []; - /* - Creating a map of the following structure - owned_objects: { - dataset_0x456: { - ot_objects: [ot_object_0x789, ...] - ..., - }, - ... - } - */ - data.forEach((obj) => { - if (owned_objects[obj.data_set_id]) { - owned_objects[obj.data_set_id].ot_objects.push({ - id: obj.ot_json_object_id, - price: obj.price, - sales: obj.sales, - }); - owned_objects[obj.data_set_id].total_sales.iadd(new BN(obj.sales, 10)); - owned_objects[obj.data_set_id].total_price.iadd(new BN(obj.price, 10)); - } else { - allDatasets.push(obj.data_set_id); - owned_objects[obj.data_set_id] = {}; - owned_objects[obj.data_set_id].ot_objects = [{ - id: obj.ot_json_object_id, - price: obj.price, - sales: obj.sales, - }]; - owned_objects[obj.data_set_id].total_sales = new BN(obj.sales, 10); - owned_objects[obj.data_set_id].total_price = new BN(obj.price, 10); - } - }); - - const allMetadata = await this.importService.getMultipleDatasetMetadata(allDatasets); - - const dataInfos = await Models.data_info.findAll({ - where: { - data_set_id: { - [Models.sequelize.Op.in]: allDatasets, - }, - }, - }); - - allDatasets.forEach((datasetId) => { - const { datasetHeader } = allMetadata.find(metadata => metadata._key === datasetId); - const dataInfo = dataInfos.find(info => info.data_set_id === datasetId); - owned_objects[datasetId].metadata = { - datasetTitle: datasetHeader.datasetTitle, - datasetTags: datasetHeader.datasetTags, - datasetDescription: datasetHeader.datasetDescription, - timestamp: dataInfo.import_timestamp, - }; - }); - - for (const dataset in owned_objects) { - result.push({ - timestamp: (new Date(owned_objects[dataset].metadata.timestamp)).getTime(), - dataset: { - id: dataset, - name: owned_objects[dataset].metadata.datasetTitle, - description: owned_objects[dataset].metadata.datasetDescription || 'No description given', - tags: owned_objects[dataset].metadata.datasetTags, - }, - ot_objects: owned_objects[dataset].ot_objects, - total_sales: owned_objects[dataset].total_sales.toString(), - total_price: owned_objects[dataset].total_price.toString(), - }); - } - } - - res.status(200); - res.send(result); - } - - async _getPermissionedDataPrice(req, res) { - this.logger.api('POST: Get permissioned data price.'); - if (req.body == null - || req.body.data_set_id == null - || req.body.seller_node_id == null - || req.body.ot_object_id == null) { - res.status(400); - res.send({ message: 'Params data_set_id, seller_node_id and ot_json_object_id are required.' }); - } - - const { - data_set_id, seller_node_id, ot_object_id, - } = req.body; - const inserted_object = await Models.handler_ids.create({ - data: JSON.stringify({ - data_set_id, seller_node_id, ot_object_id, - }), - status: 'PENDING', - }); - - const handlerId = inserted_object.dataValues.handler_id; - - await this.dvController.sendPermissionedDataPriceRequest( - data_set_id, - seller_node_id, - ot_object_id, - handlerId, - ); - - res.status(200); - res.send({ - handler_id: handlerId, - }); - } - - async _updatePermissionedDataPrice(req, res) { - this.logger.api('POST: Set permissioned data price.'); - if (req.body == null - || req.body.data_set_id == null - || req.body.ot_object_ids == null) { - res.status(400); - res.send({ message: 'Params data_set_id and ot_object_ids are required.' }); - return; - } - - const promises = []; - req.body.ot_object_ids.forEach((ot_object) => { - promises.push(new Promise(async (accept, reject) => { - const condition = { - seller_erc_id: this.config.erc725Identity.toLowerCase(), - data_set_id: req.body.data_set_id.toLowerCase(), - ot_json_object_id: ot_object.id, - }; - - const data = await Models.data_sellers.findOne({ - where: condition, - }); - - if (data) { - await Models.data_sellers.update( - { price: ot_object.price_in_trac }, - { where: { id: data.id } }, - ); - accept(); - } else { - reject(); - } - })); - }); - await Promise.all(promises).then(() => { - res.status(200); - res.send({ status: 'COMPLETED' }); - }); - } - - async _getTradingData(req, res) { - this.logger.api('GET: Get trading data.'); - const requestedType = req.params.type; - if (!requestedType || !this.trading_types.includes(requestedType)) { - res.status(400); - res.send({ - message: 'Param type with values: PURCHASED, SOLD or ALL is required.', - }); - } - const normalizedIdentity = Utilities.normalizeHex(this.config.erc725Identity); - const whereCondition = {}; - if (requestedType === this.trading_type_purchased) { - whereCondition.buyer_erc_id = normalizedIdentity; - } else if (requestedType === this.trading_type_sold) { - whereCondition.seller_erc_id = normalizedIdentity; - } - - const tradingData = await Models.data_trades.findAll({ - where: whereCondition, - order: [ - ['timestamp', 'DESC'], - ], - }); - - const allDatasets = tradingData.map(element => element.data_set_id) - .filter((value, index, self) => self.indexOf(value) === index); - - const allMetadata = await this.importService.getMultipleDatasetMetadata(allDatasets); - - const returnArray = []; - tradingData.forEach((element) => { - const { datasetHeader } = - allMetadata.find(metadata => metadata._key === element.data_set_id); - const type = normalizedIdentity === element.buyer_erc_id ? 'PURCHASED' : 'SOLD'; - returnArray.push({ - data_set: { - id: element.data_set_id, - name: datasetHeader.datasetTitle, - description: datasetHeader.datasetDescription, - tags: datasetHeader.datasetTags, - }, - ot_json_object_id: element.ot_json_object_id, - buyer_erc_id: element.buyer_erc_id, - buyer_node_id: element.buyer_node_id, - seller_erc_id: element.seller_erc_id, - seller_node_id: element.seller_node_id, - price_in_trac: element.price_in_trac, - purchase_id: element.purchase_id, - timestamp: element.timestamp, - type, - status: element.status, - }); - }); - - res.status(200); - res.send(returnArray); - } } diff --git a/ot-node.js b/ot-node.js index 934a5808d..b43777ccf 100644 --- a/ot-node.js +++ b/ot-node.js @@ -48,6 +48,7 @@ const M5ArangoPasswordMigration = require('./modules/migration/m5-arango-passwor const ImportWorkerController = require('./modules/worker/import-worker-controller'); const ImportService = require('./modules/service/import-service'); const OtJsonUtilities = require('./modules/OtJsonUtilities'); +const PermissionedDataService = require('./modules/service/permissioned-data-service'); const semver = require('semver'); @@ -296,6 +297,7 @@ class OTNode { challengeService: awilix.asClass(ChallengeService).singleton(), importWorkerController: awilix.asClass(ImportWorkerController).singleton(), importService: awilix.asClass(ImportService).singleton(), + permissionedDataService: awilix.asClass(PermissionedDataService).singleton(), }); const blockchain = container.resolve('blockchain'); await blockchain.initialize(); diff --git a/package.json b/package.json index 34798aa58..afb431ba2 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "origintrail_node", - "version": "4.1.8", + "version": "4.1.9", "description": "OriginTrail node", "main": ".eslintrc.js", "config": { diff --git a/test/bdd/features/datalayer.feature b/test/bdd/features/datalayer.feature index e4cce4f70..ce0d22842 100644 --- a/test/bdd/features/datalayer.feature +++ b/test/bdd/features/datalayer.feature @@ -175,7 +175,7 @@ Feature: Data layer related features Given DC node makes local query with previous json query Then response should contain only last imported data set id - @third + @second Scenario: Graph level data encryption Given the replication difficulty is 0 And I setup 4 nodes @@ -236,4 +236,81 @@ Feature: Data layer related features Given DC initiates the replication for last imported dataset And I wait for replications to finish Given DV publishes query consisting of path: "identifiers.id", value: "urn:epc:id:sgtin:Batch_1" and opcode: "EQ" to the network - Then all nodes with last import should answer to last network query by DV \ No newline at end of file + Then all nodes with last import should answer to last network query by DV + + + + @third + Scenario: Cover message routing protocol via proxy nodes + Given the replication difficulty is 0 + And I setup 10 nodes + And I override configuration for 3rd node + | dh_price_factor | 100 | + And I override configuration for 4th node + | dh_price_factor | 100 | + And I override configuration for 5th node + | dh_price_factor | 100 | + And I override configuration for 6th node + | dh_price_factor | 100 | + And I override configuration for 7th node + | dh_price_factor | 100 | + And I override configuration for 8th node + | dh_price_factor | 100 | + And I setup 1st node kademlia identity + | privateKey | 22bb0a45a5ab03cea8a68c991a0b5159c9edcda163840a7ed868f16cfda6a41a | + | nonce | 3 | + | proof | b58510009b511e0050843400bf603c00a3690e0069033e0003210a00252e1600e8531500a8d91f0025c10e00c3a32200e85a1000bda53c00a4c51a00bb8f300020dd02001be82400fe9932002dc13b00bf982f00a0d73f008ff51f00b814360025471600859b39008e0b2300115f3b00797b1100a52f350039db14002cc02900 | + And I setup 2nd node kademlia identity + | privateKey | e50d93f4015e55884e5ec8cb565ce8cf93d9d34a890fb91823b29599f2de6ad9 | + | nonce | 3 | + | proof | 5cf60c00dc651c00b9f80700646018002dbf1900db63240064441100b1573700e3be1700eba919009b441700c08b180028c50a0058212a0074f20200bda60e008f220900ffb11700bc061d00fcfa3c00947f1600549224007b1c060003551300be7d0d0007ba1d0068531b00db8b2c0089de0000f1bf1200276814007b042100 | + And I setup 3rd node kademlia identity + | privateKey | 6a3cb3261d109dd5bad9cbedd25d30cfe04adea1468097b538ad1272b453b008 | + | nonce | 2 | + | proof | 482e0600e61a2f006258150083453600d8a213007a97330018072a0044673a00d5ce11004ab93200a6461f00f3f43a0010640900b17f1c000e980c000aaa35004f61090059e11900469005009d3a2a007c7c1900bb5e3a0084120700c9371300990d130085de1f00967f1100127a2600c23903005bd71700ff2c000071190800 | + And I setup 4th node kademlia identity + | privateKey | 6114c3e2d542083668e154114bb8b4463cdb03860ee68788f2cb2a3375fd18f9 | + | nonce | 9 | + | proof | 9f1a0a00550e1a002dac1a0062c9230007d938009ed83900b2610100cd58350007c40c00fbc11c006ff020003a933f0091b5150005fb2200c41014000a8e1a00191c16009737220078fa10009268210002940400040c060054542e00c2f93f00524d070080f91a0072550c00ffbc3d002f590a00c5162a00cbb815006c163000 | + And I setup 5th node kademlia identity + | privateKey | f4a6c4b1108e7acd50ec00517025a877bde1215a69f235061bff6d3a120c1497 | + | nonce | 9 | + | proof | a30d0600b55e3000bf1c0a00d2f732008683170021f7180052141000935812002a580800117e150013642200c6cd2d00c9182500eb8b3c00e7df2b003c1936007931060063612b00081500001fff1900c7363700df1b3900f1450f0078603600d21a0000e2a20200419510008605140063c30e0011e42800371e0b0051963e00 | + And I setup 6th node kademlia identity + | privateKey | 32ddf611f0eee6b7d648b0135f8c2d81ecbf77c27ff72dcc19e6624977610afa | + | nonce | 2 | + | proof | 363804006e3e040025ac08007d922c0064000e000303310088de1d00a8f22700ad9825007bdf3100f3332000073c2600e3a21500b5a12a00d29713002016190090d918009de9330043b91800755e32001860020093860700ddaf0100fed92d0094c22700dcb5340095b21100c1b21a00c376170048511c00b14019005f422200 | + And I setup 7th node kademlia identity + | privateKey | 2c4e5e9b5df854f4dbc09218804666f5019b19feddf92ee9af0693fd372772e5 | + | nonce | 6 | + | proof | 2f582f003c63340040990a00d0f92100eec30d0026c91b00a9d51200194e23009f5b0200e6af2300a8f61b00ac671c00db6e1200e6801800ba310d0067b51f0051110500db7a1e00d6a71900db733e0060a213006cb4340050890f00670419000f0d0d00900e240011ce2300e9e02f0042b6120047c922001c300400ab802500 | + And I setup 8th node kademlia identity + | privateKey | 422c3249a8d4cd234a7749aafd6cb16ba0473efafaa4852a3eda499156cd551c | + | nonce | 2 | + | proof | 73ba15008e8639006f8c15007fec3400e57c020067e425005ad00a004a291600dfcc0500c6490900a893090046e237002c462a0098003b00c7de1400cc8025007bac0200d9430300c4dd0000e8a62c0090762900f18b2b001b69190031df1a0044b504000db9080091910100772a1b002e120600ad411f0027b71a00e41a2300 | + And I setup 9th node kademlia identity + | privateKey | 0c889c345307126ec08d1945c8c814696953125b73178370a255590a70c955c2 | + | nonce | 7 | + | proof | 64592d00fee93b005495260055b93b000b552c0080bb3a00a0ec030030211800ec0d0c00bae41d0078da1800d04d28000a1e130033062800c9881d002bed2d00138e1f00e15a2b00a71e1600b28b2a003c7d170055cd2a0000120d007d051300e6c40d00dc0b1d00e3ec3100521632003efa0e007fc61b0096b80f0024422900 | + And I setup 10th node kademlia identity + | privateKey | a1560bd71fc90e95802cfad2cfb0b6412a2f4fdc311351c4cbee460c01ef7e0c | + | nonce | 2 | + | proof | f9e30d00e6fc33002fe50200d46e1c00deff1600cecf3800d72307002e0b360022480f007624130030660b00981c2f00736c1200788b1e005224000060651200aeb81c0005352000bc660d0046191600bf4a06007bdb2a00f74e12005de7230034f91e0074c32d007c441a00582c2f003972050092f609008e40190064a82500 | + And I start the 2nd node + And I start the 3rd node + And I start the 4th node + And I start the 5th node + And I start the 6th node + And I start the 7th node + And I start the 8th node + And I start the 9th node + And I start the 1st node + And I stop the 1st node + And I start the 10th node + And I start the 1st node + And I use 1st node as DC + And DC imports "importers/xml_examples/Retail/01_Green_to_pink_shipment.xml" as GS1-EPCIS + And DC waits for import to finish + Given DC initiates the replication for last imported dataset + And DC waits for public key request + And I wait for replications to finish \ No newline at end of file diff --git a/test/bdd/features/permissioned_data.feature b/test/bdd/features/permissioned_data.feature index 75132b373..aad43a97a 100644 --- a/test/bdd/features/permissioned_data.feature +++ b/test/bdd/features/permissioned_data.feature @@ -3,7 +3,7 @@ Feature: Permissioned data features Given the blockchain is set up And 1 bootstrap is running - @fourth + @second Scenario: Whitelisted viewer should receive the complete dataset after query Given the replication difficulty is 1 And I setup 4 nodes @@ -26,3 +26,26 @@ Feature: Permissioned data features When DV exports the last imported dataset as OT-JSON And DV waits for export to finish Then the last import should be the same on DC and DV nodes + + + @first + Scenario: Basic purchase scenario where buyer should receive private data while seller should take payment + Given the replication difficulty is 1 + And I setup 4 nodes + And I start the nodes + And I use 1st node as DC + And DC imports "importers/use_cases/marketplace/permissioned_data_simple_sample.json" as OT-JSON + And DC waits for import to finish + Given DC initiates the replication for last imported dataset + And I wait for replications to finish + Given I use 2nd node as DV + And DV gets the list of available datasets for trading + And DV gets the price for the last imported dataset + And DV initiates purchase for the last imported dataset and waits for confirmation + And DV waits for purchase to finish +# And DC waits to take a payment + When DC exports the last imported dataset as OT-JSON + And DC waits for export to finish + When DV exports the last imported dataset as OT-JSON + And DV waits for export to finish + Then the last import should be the same on DC and DV nodes diff --git a/test/bdd/steps/endpoints.js b/test/bdd/steps/endpoints.js index c076c7227..3ab42d2aa 100644 --- a/test/bdd/steps/endpoints.js +++ b/test/bdd/steps/endpoints.js @@ -6,6 +6,7 @@ const { const BN = require('bn.js'); const { expect } = require('chai'); const fs = require('fs'); +const sleep = require('sleep-async')().Promise; const httpApiHelper = require('./lib/http-api-helper'); @@ -377,6 +378,103 @@ Given( }, ); +Given(/^([DC|DH|DV]+) gets the list of available datasets for trading$/, async function (viewer) { + this.logger.log(`${viewer} gets the list of available datasets for trading.`); + expect(viewer, 'Node type can only be DC, DH, DV.').to.be.oneOf(['DC', 'DH', 'DV']); + + const host = this.state[viewer.toLowerCase()].state.node_rpc_url; + + const availableResponse = await httpApiHelper.apiPermissionedDataAvailable(host); + expect(availableResponse[0], 'Should have keys called dataset, ot_objects, seller_erc_id, seller_node_id, timestamp') + .to.have.all.keys('dataset', 'ot_objects', 'seller_erc_id', 'seller_node_id', 'timestamp'); + + const { dataset, ot_objects, seller_node_id } = availableResponse[0]; + expect(this.state.lastImport.data.dataset_id).to.be.equal(dataset.id); + this.state.availablePurchase = { + data_set_id: dataset.id, + seller_node_id, + ot_object_id: ot_objects[0], + }; +}); + +Given(/^([DC|DH|DV]+) gets the price for the last imported dataset$/, async function (viewer) { + this.logger.log(`${viewer} gets the price for the last imported dataset.`); + expect(viewer, 'Node type can only be DC, DH, DV.').to.be.oneOf(['DC', 'DH', 'DV']); + + const host = this.state[viewer.toLowerCase()].state.node_rpc_url; + + const { handler_id } = await httpApiHelper.apiPermissionedDataGetPrice(host, this.state.availablePurchase); + await sleep.sleep(2000); + const response = await httpApiHelper.apiPermissionedDataGetPriceResult(host, handler_id); + + expect(response, 'Should have keys called data and status').to.have.all.keys('data', 'status'); + expect(response.status).to.be.equal('COMPLETED'); +}); + + +Given(/^([DC|DH|DV]+) initiates purchase for the last imported dataset and waits for confirmation$/, async function (viewer) { + this.logger.log(`${viewer} initiates purchase for the last imported dataset and waits for confirmation.`); + expect(viewer, 'Node type can only be DC, DH, DV.').to.be.oneOf(['DC', 'DH', 'DV']); + + const host = this.state[viewer.toLowerCase()].state.node_rpc_url; + + const { handler_id } = await httpApiHelper.apiPermissionedDataPurchase(host, this.state.availablePurchase); + this.state.lastPurchaseHandler = handler_id; + + this.state.lastQueryNetworkId = {}; + this.state[viewer.toLowerCase()].state.purchasedDatasets = {}; + this.state[viewer.toLowerCase()].state.purchasedDatasets[this.state.availablePurchase.data_set_id] = {}; + + const source = this.state.dc; + + const promise = new Promise((acc, reject) => { + source.once('purchase-confirmed', async (data) => { + const target = this.state[viewer.toLowerCase()]; + if (target.state.identity === data.dv_identity) { acc(); } else { reject(); } + }); + }); + + return promise; +}); + +Given(/^(DC|DV|DV2) waits for purchase to finish$/, { timeout: 300000 }, async function (targetNode) { + this.logger.log(`${targetNode} waits for purchase to finish.`); + expect(targetNode, 'Node type can only be DC, DH or DV.').to.satisfy(val => (val === 'DC' || val === 'DV2' || val === 'DV')); + expect(this.state.nodes.length, 'No started nodes').to.be.greaterThan(0); + expect(this.state.bootstraps.length, 'No bootstrap nodes').to.be.greaterThan(0); + + const source = this.state[targetNode.toLowerCase()]; + + const promise = new Promise((acc) => { + source.once('purchase-completed', async () => { + acc(); + }); + }); + + + return promise; +}); + + +Given(/^(DC|DV|DV2) waits to take a payment$/, { timeout: 300000 }, async function (targetNode) { + this.logger.log(`${targetNode} waits to take a payment.`); + expect(targetNode, 'Node type can only be DC, DH or DV.').to.satisfy(val => (val === 'DC' || val === 'DV2' || val === 'DV')); + expect(this.state.nodes.length, 'No started nodes').to.be.greaterThan(0); + expect(this.state.bootstraps.length, 'No bootstrap nodes').to.be.greaterThan(0); + + const source = this.state[targetNode.toLowerCase()]; + + const promise = new Promise((acc) => { + source.once('purchase-payment-taken', async () => { + acc(); + }); + }); + + + return promise; +}); + + Given(/^default initial token amount should be deposited on (\d+)[st|nd|rd|th]+ node's profile$/, async function (nodeIndex) { expect(nodeIndex, 'Invalid index.').to.be.within(0, this.state.nodes.length); diff --git a/test/bdd/steps/lib/http-api-helper.js b/test/bdd/steps/lib/http-api-helper.js index c1cae2e02..b4001a728 100644 --- a/test/bdd/steps/lib/http-api-helper.js +++ b/test/bdd/steps/lib/http-api-helper.js @@ -728,6 +728,115 @@ async function apiNodeInfo(nodeRpcUrl) { }); } +/** + * Fetch api/latest/permissioned_data/available information + * + * @param {string} nodeRpcUrl URL in following format http://host:port + * @return {Promise.} + */ +async function apiPermissionedDataAvailable(nodeRpcUrl) { + return new Promise((accept, reject) => { + request( + { + method: 'GET', + headers: { 'Content-Type': 'application/json' }, + uri: `${nodeRpcUrl}/api/latest/permissioned_data/available`, + json: true, + }, + (err, res, body) => { + if (err) { + reject(err); + return; + } + accept(body); + }, + ); + }); +} + +/** + * Fetch api/latest/network/permissioned_data/get_price + * + * @param {string} nodeRpcUrl URL in following format http://host:port + * @param {object} params body + * @return {Promise.} + */ +async function apiPermissionedDataGetPrice(nodeRpcUrl, params) { + return new Promise((accept, reject) => { + request( + { + method: 'POST', + body: params, + headers: { 'Content-Type': 'application/json' }, + uri: `${nodeRpcUrl}/api/latest/network/permissioned_data/get_price`, + json: true, + }, + (err, res, body) => { + if (err) { + reject(err); + return; + } + accept(body); + }, + ); + }); +} + + +/** + * Fetch /api/latest/network/permissioned_data/get_price/result + * @typedef {Object} ImportResult + * + * @param {string} nodeRpcUrl URL in following format http://host:port + * @param {string} handler_id + * @return {Promise.} + */ +async function apiPermissionedDataGetPriceResult(nodeRpcUrl, handler_id) { + return new Promise((accept, reject) => { + request({ + method: 'GET', + headers: { 'Content-Type': 'application/json' }, + url: `${nodeRpcUrl}/api/latest/network/permissioned_data/get_price/result/${handler_id}`, + json: true, + }, (error, response, body) => { + if (error) { + reject(error); + return; + } + accept(body); + }); + }); +} + +/** + * Fetch api/latest/network/permissioned_data/purchase + * + * @param {string} nodeRpcUrl URL in following format http://host:port + * @param {object} params body + * @return {Promise.} + */ +async function apiPermissionedDataPurchase(nodeRpcUrl, params) { + return new Promise((accept, reject) => { + request( + { + method: 'POST', + body: params, + headers: { 'Content-Type': 'application/json' }, + uri: `${nodeRpcUrl}/api/latest/network/permissioned_data/purchase`, + json: true, + }, + (err, res, body) => { + if (err) { + reject(err); + return; + } + accept(body); + }, + ); + }); +} + + /** * @typedef {Object} ApiBalanceInfo * @property {Object} profile info about profile balance @@ -792,4 +901,8 @@ module.exports = { apiGetDatasetInfo, apiQueryNetworkReadAndExport, apiWhitelistViewer, + apiPermissionedDataAvailable, + apiPermissionedDataGetPrice, + apiPermissionedDataGetPriceResult, + apiPermissionedDataPurchase, }; diff --git a/test/bdd/steps/lib/local-blockchain.js b/test/bdd/steps/lib/local-blockchain.js index 003736c6f..25e1e5b72 100644 --- a/test/bdd/steps/lib/local-blockchain.js +++ b/test/bdd/steps/lib/local-blockchain.js @@ -155,6 +155,8 @@ class LocalBlockchain { const litigationSource = fs.readFileSync(path.join(__dirname, '../../../../modules/Blockchain/Ethereum/contracts/Litigation.sol'), 'utf8'); const litigationStorageSource = fs.readFileSync(path.join(__dirname, '../../../../modules/Blockchain/Ethereum/contracts/LitigationStorage.sol'), 'utf8'); const replacementSource = fs.readFileSync(path.join(__dirname, '../../../../modules/Blockchain/Ethereum/contracts/Replacement.sol'), 'utf8'); + const marketplaceStorageSource = fs.readFileSync(path.join(__dirname, '../../../../modules/Blockchain/Ethereum/contracts/MarketplaceStorage.sol'), 'utf8'); + const marketplaceSource = fs.readFileSync(path.join(__dirname, '../../../../modules/Blockchain/Ethereum/contracts/Marketplace.sol'), 'utf8'); this.contracts = {}; @@ -182,6 +184,8 @@ class LocalBlockchain { 'Litigation.sol': litigationSource, 'LitigationStorage.sol': litigationStorageSource, 'Replacement.sol': replacementSource, + 'MarketplaceStorage.sol': marketplaceStorageSource, + 'Marketplace.sol': marketplaceSource, }, }, 1); @@ -244,6 +248,17 @@ class LocalBlockchain { this.contracts.Identity.data = `0x${compileResult.contracts['Identity.sol:Identity'].bytecode}`; this.contracts.Identity.abi = JSON.parse(compileResult.contracts['Identity.sol:Identity'].interface); this.contracts.Identity.artifact = new this.web3.eth.Contract(this.contracts.Identity.abi); + + + this.contracts.MarketplaceStorage = {}; + this.contracts.MarketplaceStorage.data = `0x${compileResult.contracts['MarketplaceStorage.sol:MarketplaceStorage'].bytecode}`; + this.contracts.MarketplaceStorage.abi = JSON.parse(compileResult.contracts['MarketplaceStorage.sol:MarketplaceStorage'].interface); + this.contracts.MarketplaceStorage.artifact = new this.web3.eth.Contract(this.contracts.MarketplaceStorage.abi); + + this.contracts.Marketplace = {}; + this.contracts.Marketplace.data = `0x${compileResult.contracts['Marketplace.sol:Marketplace'].bytecode}`; + this.contracts.Marketplace.abi = JSON.parse(compileResult.contracts['Marketplace.sol:Marketplace'].interface); + this.contracts.Marketplace.artifact = new this.web3.eth.Contract(this.contracts.Marketplace.abi); } async deployContracts() { @@ -370,6 +385,27 @@ class LocalBlockchain { .send({ from: accounts[7], gas: 3000000 }) .on('error', console.error); + this.logger.log('Deploying MarketplaceStorage contract'); + [this.contracts.MarketplaceStorage.deploymentReceipt, this.contracts.MarketplaceStorage.instance] = await this._deployContract( + this.web3, this.contracts.MarketplaceStorage.artifact, this.contracts.MarketplaceStorage.data, + [this.contracts.Hub.instance._address], accounts[7], + ); + + await this.contracts.Hub.instance.methods.setContractAddress('MarketplaceStorage', this.contracts.MarketplaceStorage.instance._address) + .send({ from: accounts[7], gas: 3000000 }) + .on('error', console.error); + + + this.logger.log('Deploying Marketplace contract'); + [this.contracts.Marketplace.deploymentReceipt, this.contracts.Marketplace.instance] = await this._deployContract( + this.web3, this.contracts.Marketplace.artifact, this.contracts.Marketplace.data, + [this.contracts.Hub.instance._address], accounts[7], + ); + + await this.contracts.Hub.instance.methods.setContractAddress('Marketplace', this.contracts.Marketplace.instance._address) + .send({ from: accounts[7], gas: 3000000 }) + .on('error', console.error); + // Deploy tokens. const amountToMint = '50000000000000000000000000'; // 5e25 const amounts = []; diff --git a/test/bdd/steps/lib/otnode.js b/test/bdd/steps/lib/otnode.js index a13423ab5..0845ca728 100644 --- a/test/bdd/steps/lib/otnode.js +++ b/test/bdd/steps/lib/otnode.js @@ -230,6 +230,8 @@ class OtNode extends EventEmitter { this.state.node_url = line.substr(line.search('OT Node listening at ') + 'OT Node listening at '.length, line.length - 1); } else if (line.match(/Import complete/gi)) { this.emit('import-complete'); + } else if (line.match(/Public key request received/gi)) { + this.emit('public-key-request'); } else if (line.match(/Export complete.*/gi)) { this.emit('export-complete'); } else if (line.match(/.*\[DH] Replication finished for offer ID .+/gi)) { @@ -414,6 +416,14 @@ class OtNode extends EventEmitter { } else if (line.match(/Replication request from holder identity .+ declined! Unacceptable reputation: .+./gi)) { const dhIdentity = line.match(identityWithPrefixRegex)[0]; this.state.declinedDhIdentity = dhIdentity; + } else if (line.match(/Purchase confirmed for ot_object .+ received from .+\. Sending purchase response\./gi)) { + const ot_object_id = line.match(new RegExp('ot_object (.*) received'))[1]; + const dv_identity = line.match(new RegExp('from (.*)\\. Sending purchase response\\.'))[1]; + this.emit('purchase-confirmed', { ot_object_id, dv_identity }); + } else if (line.match(/Purchase .+ completed\. Data stored successfully/gi)) { + this.emit('purchase-completed'); + } else if (line.match(/Payment has been taken for purchase .+/gi)) { + this.emit('purchase-payment-taken'); } } diff --git a/test/bdd/steps/network.js b/test/bdd/steps/network.js index 5bb5077b1..9517a1fff 100644 --- a/test/bdd/steps/network.js +++ b/test/bdd/steps/network.js @@ -510,6 +510,23 @@ Then(/^the last two datasets should have the same hashes$/, async function () { .to.be.equal(this.state.secondLastImport.data.dataset_id); }); +Given(/^(DC|DV|DV2) waits for public key request$/, { timeout: 120000 }, async function (targetNode) { + this.logger.log(`${targetNode} waits for public key request.`); + expect(targetNode, 'Node type can only be DC, DH or DV.').to.satisfy(val => (val === 'DC' || val === 'DV2' || val === 'DV')); + expect(this.state.nodes.length, 'No started nodes').to.be.greaterThan(0); + expect(this.state.bootstraps.length, 'No bootstrap nodes').to.be.greaterThan(0); + + const target = this.state[targetNode.toLowerCase()]; + + const promise = new Promise((acc) => { + target.once('public-key-request', async () => { + acc(); + }); + }); + + return promise; +}); + Given(/^I wait for replication[s] to finish$/, { timeout: 1800000 }, function () { this.logger.log('I wait for replication to finish'); expect(!!this.state.dc, 'DC node not defined. Use other step to define it.').to.be.equal(true); @@ -1032,6 +1049,15 @@ Given(/^API calls will not be authorized/, { timeout: 180000 }, function (done) Promise.all(promises).then(() => done()); }); +Given(/^I override configuration for (\d+)[st|nd|rd|th]+ node*$/, { timeout: 120000 }, function (nodeIndex, configuration, done) { + const configurationOverride = unpackRawTable(configuration); + const node = this.state.nodes[nodeIndex - 1]; + node.overrideConfiguration(configurationOverride); + this.logger.log(`Configuration updated for node ${node.id}`); + done(); +}); + + Given(/^I override configuration for all nodes*$/, { timeout: 120000 }, function (configuration, done) { const configurationOverride = unpackRawTable(configuration); @@ -1042,6 +1068,13 @@ Given(/^I override configuration for all nodes*$/, { timeout: 120000 }, function done(); }); + +Given(/^I setup (\d+)[st|nd|rd|th]+ node kademlia identity*$/, { timeout: 120000 }, function (nodeIndex, configuration, done) { + configuration = unpackRawTable(configuration); + fs.writeFileSync(`${this.state.nodes[nodeIndex - 1].options.configDir}/identity.json`, JSON.stringify(configuration)); + done(); +}); + Given(/^I override configuration using variables for all nodes*$/, { timeout: 120000 }, function (configuration, done) { const configurationOverride = configuration.raw(); for (const node of this.state.nodes) { diff --git a/test/modules/service/permissioned-data-service-test.js b/test/modules/service/permissioned-data-service-test.js index 58b0b8953..96088d58e 100644 --- a/test/modules/service/permissioned-data-service-test.js +++ b/test/modules/service/permissioned-data-service-test.js @@ -1,201 +1,394 @@ -const { describe, before, it } = require('mocha'); +const { + describe, before, beforeEach, it, +} = require('mocha'); const { assert, expect } = require('chai'); const Web3 = require('web3'); +const ImportUtilities = require('../../../modules/ImportUtilities'); +const Utilities = require('./../../../modules/Utilities'); const Storage = require('../../../modules/Storage'); const models = require('../../../models'); +const sample_data = require('../test_data/otjson-graph'); +const defaultConfig = require('../../../config/config.json').mainnet; +const rc = require('rc'); +const pjson = require('../../../package.json'); +const logger = require('../../../modules/logger'); +const awilix = require('awilix'); +const PermissionedDataService = require('../../../modules/service/permissioned-data-service'); +const MerkleTree = require('../../../modules/Merkle'); +const crypto = require('crypto'); +const abi = require('ethereumjs-abi'); +const Encryption = require('../../../modules/RSAEncryption'); +const samplePermissionedObject = { + properties: { + permissioned_data: { + data: { + 'urn:ot:object:product:batch:humidity': '19.7', + 'urn:ot:object:product:batch:power_feeding': '85', + 'urn:ot:object:product:batch:productId': 'urn:ot:object:actor:id:KakaxiSN687', + 'urn:ot:object:product:batch:rainfall': '0.0', + 'urn:ot:object:product:batch:solar_radiation': '0.0', + 'urn:ot:object:product:batch:temperature': '22.0', + vocabularyType: 'urn:ot:object:batch', + }, + }, + }, +}; + +let config; +let permissionedDataService; + +class GraphStorageMock { + constructor(log) { + this.logger = log; + } +} + describe('Permission data service test', () => { + beforeEach('Setup container', async () => { + // Create the container and set the injectionMode to PROXY (which is also the default). + process.env.NODE_ENV = 'mainnet'; + const container = awilix.createContainer({ + injectionMode: awilix.InjectionMode.PROXY, + }); + + const graphStorage = new GraphStorageMock(logger); + + config = rc(pjson.name, defaultConfig); + container.register({ + config: awilix.asValue(config), + logger: awilix.asValue(logger), + graphStorage: awilix.asValue(graphStorage), + }); + permissionedDataService = new PermissionedDataService(container.cradle); + }); + + it('Calculate the public data hash', () => { + const originalGraph = Utilities + .copyObject(sample_data.permissioned_data_graph); + ImportUtilities.calculateGraphPermissionedDataHashes(originalGraph['@graph']); + + const shuffledGraph = Utilities + .copyObject(sample_data.permissioned_data_graph_shuffled); + ImportUtilities.calculateGraphPermissionedDataHashes(shuffledGraph['@graph']); + + const differentGraph = Utilities + .copyObject(sample_data.permissioned_data_graph_2); + ImportUtilities.calculateGraphPermissionedDataHashes(differentGraph['@graph']); + + const originalGraphRootHash = ImportUtilities.calculateGraphPublicHash(originalGraph); + const shuffledGraphRootHash = ImportUtilities.calculateGraphPublicHash(shuffledGraph); + const differentGraphRootHash = ImportUtilities.calculateGraphPublicHash(differentGraph); + + assert(originalGraphRootHash != null); + assert(shuffledGraphRootHash != null); + assert(differentGraphRootHash != null); + + // Hashes should be 32 Bytes (64 characters) with 0x preceding the hash, so 66 characters + assert(typeof originalGraphRootHash === 'string'); + assert(typeof shuffledGraphRootHash === 'string'); + assert(typeof differentGraphRootHash === 'string'); + + assert(originalGraphRootHash.length === 66); + assert(shuffledGraphRootHash.length === 66); + assert(differentGraphRootHash.length === 66); + + assert.notEqual( + originalGraphRootHash, + shuffledGraphRootHash, + 'Graph root hash for same object with attributes in different order!', + ); + + assert.notEqual( + originalGraphRootHash, + differentGraphRootHash, + 'Graph root hash for different objects is the same!', + ); + }); + + it('Should correctly reconstruct encoded object', () => { + const { + permissioned_data_original_length, permissioned_data_array_length, key, + encoded_data, permissioned_data_root_hash, encoded_data_root_hash, + } = permissionedDataService.encodePermissionedData(samplePermissionedObject); + + const decoded_data = permissionedDataService.decodePermissionedData( + encoded_data, + key, + ); + + const result = permissionedDataService.reconstructPermissionedData( + decoded_data, + permissioned_data_array_length, + permissioned_data_original_length, + ); + + assert.equal( + Utilities.sortedStringify(samplePermissionedObject.properties.permissioned_data.data), + Utilities.sortedStringify(result), + 'Reconstructed object is not the same as the original object', + ); + }); + + it('Should validate correct permissioned data tree with validatePermissionedDataTree', () => { + const { + permissioned_data_original_length, permissioned_data_array_length, key, + encoded_data, permissioned_data_root_hash, encoded_data_root_hash, + } = permissionedDataService.encodePermissionedData(samplePermissionedObject); + + const decodedPermissionedData = permissionedDataService + .decodePermissionedData(encoded_data, key); + + const validationResult = permissionedDataService.validatePermissionedDataTree( + decodedPermissionedData, + permissioned_data_array_length, + ); + + assert(!validationResult.error, 'Correctly encoded data returned an error.'); + }); + + it('Should report error for incorrect permissioned data tree with validatePermissionedDataTree', () => { + const { + permissioned_data_original_length, permissioned_data_array_length, key, + encoded_data, permissioned_data_root_hash, encoded_data_root_hash, + } = permissionedDataService.encodePermissionedData(samplePermissionedObject); + + const decodedPermissionedData = permissionedDataService + .decodePermissionedData(encoded_data, key); + + const decodedDataMerkleTree = ImportUtilities + .calculatePermissionedDataMerkleTree(samplePermissionedObject.properties.permissioned_data, 'purchase'); + const randomLevel = 2 + + Math.floor(Math.random() * (decodedDataMerkleTree.levels.length - 2)); + const randomLeaf = + Math.floor(Math.random() * decodedDataMerkleTree.levels[randomLevel].length); + + let corruptedIndex = randomLeaf; + let inputIndex = randomLeaf * 2; + for (let levelIndex = 1; levelIndex < randomLevel; levelIndex += 1) { + const level = decodedDataMerkleTree.levels[levelIndex]; + if (level.length % 2 === 1) { + corruptedIndex += level.length + 1; + } else { + corruptedIndex += level.length; + } + + if (levelIndex > 1) { + const previousLevel = decodedDataMerkleTree.levels[levelIndex - 1]; + if (previousLevel.length % 2 === 1) { + inputIndex += previousLevel.length + 1; + } else { + inputIndex += previousLevel.length; + } + } + } + + decodedPermissionedData[corruptedIndex] = + '0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff'; + + const validationResult = permissionedDataService.validatePermissionedDataTree( + decodedPermissionedData, + permissioned_data_array_length, + ); + + assert(validationResult.error, 'Corrupted decoded data passed validation.'); + assert.equal(validationResult.outputIndex, corruptedIndex, 'Reported output index is incorrect'); + assert.equal(validationResult.inputIndexLeft, inputIndex, 'Reported input index is incorrect'); + }); + + it('Should validate correct permissioned data decoded root hash', () => { + const { + permissioned_data_original_length, permissioned_data_array_length, key, + encoded_data, encoded_data_root_hash, + } = permissionedDataService.encodePermissionedData(samplePermissionedObject); + + const permissionedDataRootHash = ImportUtilities + .calculatePermissionedDataHash(samplePermissionedObject.properties.permissioned_data); + + const decodedPermissionedData = permissionedDataService + .decodePermissionedData(encoded_data, key); + + const rootHashMatches = permissionedDataService + .validatePermissionedDataRoot(decodedPermissionedData, permissionedDataRootHash); + + assert(rootHashMatches, 'Correct permissioned data root hash failed validation.'); + }); + + it('Should report error for incorrect permissioned data decoded root hash', () => { + const { + permissioned_data_original_length, permissioned_data_array_length, key, + encoded_data, encoded_data_root_hash, + } = permissionedDataService.encodePermissionedData(samplePermissionedObject); + + const permissionedDataRootHash = + '0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff'; + + const decodedPermissionedData = permissionedDataService + .decodePermissionedData(encoded_data, key); + + const rootHashMatches = permissionedDataService + .validatePermissionedDataRoot(decodedPermissionedData, permissionedDataRootHash); + + assert(!rootHashMatches, 'Correct permissioned data root hash failed validation.'); + }); + + it('Should generate a valid proof for incorrect data', () => { + const blocks = [ + 'A', + 'B', + 'C', + 'D', + 'E', + 'F', + ]; + + for (let i = 0; i < blocks.length; i += 1) { + blocks[i] = Buffer.from(blocks[i]).toString('hex').padStart(64, '0'); + } + + const originalMerkleTree = new MerkleTree(blocks, 'purchase', 'soliditySha3'); + + const { + key, + encoded_data, + permissioned_data_root_hash, + encoded_data_root_hash, + } = permissionedDataService._encodePermissionedDataMerkleTree(originalMerkleTree); + + encoded_data[11] = '0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff'; + + // 0 A \ + // 1 B - AB \ + // 2 C \ >- ABCD + // 3 D - CD / \ + // 4 E \ > - ABCDEFEF + // 5 F - EF >- EFEF / + // 6 AB + // 7 CD + // 8 EF + // 9 (EF) + // 10 ABCD + // 11 EFEF + // 12 ABCDEF + + const encodedMerkleTree = new MerkleTree(encoded_data, 'purchase', 'soliditySha3'); + + const permissioned_data_array_length = 6; + + const decodedPermissionedData = permissionedDataService + .decodePermissionedData(encoded_data, key); + + + const validationResult = permissionedDataService.validatePermissionedDataTree( + decodedPermissionedData, + permissioned_data_array_length, + ); + + assert(validationResult.error, 'Corrupted decoded data passed validation.'); + assert.equal(validationResult.inputIndexLeft, 8); + assert.equal(validationResult.outputIndex, 11); + + const { + encodedInputLeft, + encodedOutput, + proofOfEncodedInputLeft, + proofOfEncodedOutput, + } = permissionedDataService.prepareNodeDisputeData( + encoded_data, + validationResult.inputIndexLeft, + validationResult.outputIndex, + ); + + const outputProofResult = encodedMerkleTree.calculateProofResult( + proofOfEncodedOutput, + encodedOutput, + validationResult.outputIndex, + ); + assert.equal( + Utilities.normalizeHex(outputProofResult), + Utilities.normalizeHex(encodedMerkleTree.getRoot()), + 'Invalid Merkle proof for output element.', + ); + + const inputProofResult = encodedMerkleTree.calculateProofResult( + proofOfEncodedInputLeft, + encodedInputLeft, + validationResult.inputIndexLeft, + ); + assert.equal( + Utilities.normalizeHex(inputProofResult), + Utilities.normalizeHex(encodedMerkleTree.getRoot()), + 'Invalid Merkle proof for input element.', + ); + + const keyHash = abi.soliditySHA3(['bytes32', 'uint256'], [key, 8]); + const calculatedInput = Encryption.xor(encodedInputLeft, keyHash); + const decodedInput = decodedPermissionedData[validationResult.inputIndexLeft + 1]; + + assert.equal(calculatedInput, decodedInput, 'Decoded and manually decoded hashes do not match.'); + assert.equal(decodedInput, originalMerkleTree.levels[2][2], 'Decoded and originally submitted hashes do not match.'); + + const expectedHash = + originalMerkleTree._generateInternalHash(calculatedInput, decodedInput); + + assert.equal(expectedHash, originalMerkleTree.levels[3][1], 'Calculated and originally submitted output hashes do not match'); + + const actualHash = decodedPermissionedData[validationResult.outputIndex]; + assert.notEqual(actualHash, originalMerkleTree.levels[3][1], 'Original and corrupted output hashes match'); + }); + + it('Calculate the root hash on one permissioned data object', () => { + const originalObject = Utilities.copyObject(sample_data.permissioned_data_object); + const shuffledObject = Utilities + .copyObject(sample_data.permissioned_data_object_shuffled); + const differentObject = Utilities.copyObject(sample_data.permissioned_data_object_2); + + const originalRootHash = ImportUtilities.calculatePermissionedDataHash(originalObject); + const shuffledRootHash = ImportUtilities.calculatePermissionedDataHash(shuffledObject); + const differentRootHash = ImportUtilities.calculatePermissionedDataHash(differentObject); + + assert(originalRootHash != null); + assert(shuffledRootHash != null); + assert(differentRootHash != null); + + // Hashes should be 32 Bytes (64 characters) with 0x preceding the hash, so 66 characters + assert(typeof originalRootHash === 'string'); + assert(typeof shuffledRootHash === 'string'); + assert(typeof differentRootHash === 'string'); + + assert(originalRootHash.length === 66); + assert(shuffledRootHash.length === 66); + assert(differentRootHash.length === 66); + + assert.equal( + originalRootHash, + shuffledRootHash, + 'Permissioned data root hash for same object with attributes in different order!', + ); + + assert.notEqual( + originalRootHash, + differentRootHash, + 'Permisssioned data root hash for different objects is the same!', + ); + }); + // eslint-disable-next-line max-len + it('Calculating the root hash of an empty permissioned data object should throw an error', () => { + const testObject = {}; + + let errorHappened = false; + try { + ImportUtilities.calculatePermissionedDataHash(testObject); + } catch (e) { + errorHappened = true; + assert.equal( + e.message, + 'Cannot calculate root hash of an empty object', + 'Unexpected error received', + ); + } - // before('Setup models', async () => { - // Storage.models = (await models.sequelize.sync()).models; - // }); - // - // beforeEach('Setup ctx', async function setupCtx() { - // this.timeout(5000); - // - // const config = rc(pjson.name, defaultConfig); - // systemDb = new Database(); - // systemDb.useBasicAuth(config.database.username, config.database.password); - // - // // Drop test database if exist. - // const listOfDatabases = await systemDb.listDatabases(); - // if (listOfDatabases.includes(databaseName)) { - // await systemDb.dropDatabase(databaseName); - // } - // - // await systemDb.createDatabase( - // databaseName, - // [{ - // username: config.database.username, - // passwd: config.database.password, - // active: true, - // }], - // ); - // - // config.database.database = databaseName; - // config.erc725Identity = '0x611d771aAfaa3D6Fb66c4a81D97768300a6882D5'; - // config.node_wallet = '0xa9a07f3c53ec5de8dd83039ca27fae83408e16f5'; - // config.node_private_key = - // '952e45854ca5470a6d0b6cb86346c0e9c4f8f3a5a459657df8c94265183b9253'; - // - // // Create the container and set the injectionMode to PROXY (which is also the default). - // const container = awilix.createContainer({ - // injectionMode: awilix.InjectionMode.PROXY, - // }); - // - // const web3 = new Web3(new Web3.providers.HttpProvider('https://rinkeby.infura.io/1WRiEqAQ9l4SW6fGdiDt')); - // - // graphStorage = new GraphStorage(config.database, logger); - // container.register({ - // logger: awilix.asValue(logger), - // gs1Utilities: awilix.asClass(GS1Utilities), - // graphStorage: awilix.asValue(graphStorage), - // schemaValidator: awilix.asClass(SchemaValidator).singleton(), - // importService: awilix.asClass(ImportService).singleton(), - // epcisOtJsonTranspiler: awilix.asClass(EpcisOtJsonTranspiler).singleton(), - // remoteControl: awilix.asValue({ - // importRequestData: () => { - // }, - // }), - // network: awilix.asClass(Network), - // networkUtilities: awilix.asClass(NetworkUtilities), - // emitter: awilix.asClass(EventEmitter), - // product: awilix.asClass(Product), - // web3: awilix.asValue(web3), - // config: awilix.asValue(config), - // permissionedDataService: awilix.asClass(PermissionedDataService).singleton(), - // }); - // await graphStorage.connect(); - // importService = container.resolve('importService'); - // epcisOtJsonTranspiler = container.resolve('epcisOtJsonTranspiler'); - // }); - // - // it('Calculate the public root hash of one graph', () => { - // const originalGraph = Utilities. - // copyObject(sample_data.permissioned_data_graph['@graph']); - // ImportUtilities.calculateGraphPermissionedDataHashes(originalGraph); - // - // const shuffledGraph = Utilities. - // copyObject(sample_data.permissioned_data_graph_shuffled['@graph']); - // ImportUtilities.calculateGraphPermissionedDataHashes(shuffledGraph); - // - // const differentGraph = Utilities. - // copyObject(sample_data.permissioned_data_graph_2['@graph']); - // ImportUtilities.calculateGraphPermissionedDataHashes(differentGraph); - // - // const originalGraphRootHash = ImportUtilities.calculateGraphPublicHash(originalGraph); - // const shuffledGraphRootHash = ImportUtilities.calculateGraphPublicHash(shuffledGraph); - // const differentGraphRootHash = ImportUtilities.calculateGraphPublicHash(differentGraph); - // - // assert(originalGraphRootHash != null); - // assert(shuffledGraphRootHash != null); - // assert(differentGraphRootHash != null); - // - // // Hashes should be 32 Bytes (64 characters) with 0x preceding the hash, so 66 characters - // assert(typeof originalGraphRootHash === 'string'); - // assert(typeof shuffledGraphRootHash === 'string'); - // assert(typeof differentGraphRootHash === 'string'); - // - // assert(originalGraphRootHash.length === 66); - // assert(shuffledGraphRootHash.length === 66); - // assert(differentGraphRootHash.length === 66); - // - // assert.equal( - // originalGraphRootHash, - // shuffledGraphRootHash, - // 'Graph root hash for same object with attributes in different order!', - // ); - // - // assert.notEqual( - // originalGraphRootHash, - // differentGraphRootHash, - // 'Graph root hash for different objects is the same!', - // ); - // }); - // - // it('Encoding verification', () => { - // const permissionedObject = { - // data: { - // 'urn:ot:object:product:batch:humidity': '19.7', - // 'urn:ot:object:product:batch:power_feeding': '85', - // 'urn:ot:object:product:batch:productId': 'urn:ot:object:actor:id:KakaxiSN687', - // 'urn:ot:object:product:batch:rainfall': '0.0', - // 'urn:ot:object:product:batch:solar_radiation': '0.0', - // 'urn:ot:object:product:batch:temperature': '22.0', - // vocabularyType: 'urn:ot:object:batch', - // }, - // }; - // - // const { - // permissioned_data_original_length, permissioned_data_array_length, key, - // encoded_data, permissioned_data_root_hash, encoded_data_root_hash, - // } = ImportUtilities.encodePermissionedData(permissionedObject); - // - // const result = ImportUtilities.validateAndDecodePermissionedData( - // encoded_data, - // key, - // permissioned_data_array_length, - // permissioned_data_original_length, - // ); - // - // assert.equal( - // Utilities.sortedStringify(permissionedObject.data), - // Utilities.sortedStringify(result.permissionedData), - // ); - // }); - // - // it('Calculate the root hash on one permissioned data object', () => { - // const originalObject = Utilities.copyObject(sample_data.permissioned_data_object); - // const shuffledObject = Utilities. - // copyObject(sample_data.permissioned_data_object_shuffled); - // const differentObject = Utilities.copyObject(sample_data.permissioned_data_object_2); - // - // const originalRootHash = ImportUtilities.calculatePermissionedDataHash(originalObject); - // const shuffledRootHash = ImportUtilities.calculatePermissionedDataHash(shuffledObject); - // const differentRootHash = ImportUtilities.calculatePermissionedDataHash(differentObject); - // - // assert(originalRootHash != null); - // assert(shuffledRootHash != null); - // assert(differentRootHash != null); - // - // // Hashes should be 32 Bytes (64 characters) with 0x preceding the hash, so 66 characters - // assert(typeof originalRootHash === 'string'); - // assert(typeof shuffledRootHash === 'string'); - // assert(typeof differentRootHash === 'string'); - // - // assert(originalRootHash.length === 66); - // assert(shuffledRootHash.length === 66); - // assert(differentRootHash.length === 66); - // - // assert.equal( - // originalRootHash, - // shuffledRootHash, - // 'Permissioned data root hash for same object with attributes in different order!', - // ); - // - // assert.notEqual( - // originalRootHash, - // differentRootHash, - // 'Permisssioned data root hash for different objects is the same!', - // ); - // }); - // it('Calculating the root hash of an empty permissioned - // data object should throw an error', () => { - // const testObject = {}; - // - // let errorHappened = false; - // try { - // ImportUtilities.calculatePermissionedDataHash(testObject); - // } catch (e) { - // errorHappened = true; - // assert.equal( - // e.message, - // 'Cannot calculate root hash of an empty object', - // 'Unexpected error received', - // ); - // } - // - // assert(errorHappened, 'calculatePermissionedDataHash did not throw an error!'); - // }); + assert(errorHappened, 'calculatePermissionedDataHash did not throw an error!'); + }); });