From 1b943ce9a7c17f1fb35940c4f57dd5a9c951c98a Mon Sep 17 00:00:00 2001 From: Dhruv Bhanushali <119250923+theDRB123@users.noreply.github.com> Date: Sat, 22 Jun 2024 00:03:41 +0530 Subject: [PATCH] Revert "Decoupling mongo db dependencies to allow adding postgreSQL (#213)" --- README.md | 6 +- .../Storage/Mongo/Types/Investment.cs | 2 +- .../ProjectTransactionsSyncRunner.cs | 1 - .../Sync/SyncTasks/ProjectsSyncRunner.cs | 1 - src/Blockcore.Indexer.Angor/appsettings.json | 5 +- src/Blockcore.Indexer.Cirrus/CirrusStartup.cs | 1 - .../Operations/Types/CirrusStorageBatch.cs | 7 +- .../Storage/CirrusBlockMapping.cs | 2 +- .../Mongo/CirrusMongoStorageOperations.cs | 14 +++- .../Mongo/CirrusStorageBatchFactory.cs | 10 --- src/Blockcore.Indexer.Cirrus/appsettings.json | 6 +- .../Client/Types/PeerInfo.cs | 2 + .../Controllers/InsightController.cs | 6 +- .../Controllers/QueryController.cs | 2 +- .../Controllers/StatsController.cs | 11 +-- .../Models/BalanceForAddress.cs | 7 -- .../Models/QueryBlock.cs | 22 ++++-- .../Models/QueryBlockResults.cs | 10 --- .../Models/QueryOrphanBlock.cs | 5 +- .../Operations/IStorageOperations.cs | 2 +- .../Operations/IUtxoCache.cs | 19 +++++ .../Operations/Types/InsertStats.cs | 8 ++ .../Operations/Types/StorageBatch.cs | 17 +++-- .../Operations/Types/UtxoCache.cs | 57 ++++++++++++++ .../Settings/IndexerSettings.cs | 6 +- src/Blockcore.Indexer.Core/Startup.cs | 37 ++++++--- .../Storage/DatabaseRegistration.cs | 43 ----------- .../IMapMongoBlockToStorageBlock.cs | 2 +- .../Storage/IStorage.cs | 14 +--- .../Storage/IStorageBatchFactory.cs | 8 -- .../MapMongoBlockToStorageBlock.cs | 2 +- .../Storage/Mongo/BlockRewindOperation.cs | 1 - .../Storage/Mongo/IMongoDb.cs | 3 +- .../Storage/Mongo/MongoBuilder.cs | 10 --- .../Storage/Mongo/MongoData.cs | 75 ++++++------------- .../Storage/Mongo/MongoDb.cs | 8 +- .../Storage/Mongo/MongoStorageBatchFactory.cs | 9 --- .../Storage/Mongo/MongoStorageOperations.cs | 59 +++++++-------- .../Mongo/Types/AddressUtxoComputedTable.cs | 2 - .../Storage/Mongo/Types/InputTable.cs | 2 - .../Storage/Mongo/Types/MempoolTable.cs | 1 - .../Storage/Mongo/Types/MongoStorageBatch.cs | 43 ----------- .../Storage/{ => Mongo}/Types/Outpoint.cs | 2 +- .../Storage/Mongo/Types/OutputTable.cs | 12 ++- .../Storage/Mongo/Types/UnspentOutputTable.cs | 2 - .../Storage/Types/Output.cs | 14 ---- .../Storage/Types/PeerDetails.cs | 8 -- .../Sync/SyncOperations.cs | 21 ++++-- .../SyncTasks/BlockIndexer.cs} | 16 ++-- .../Sync/SyncTasks/BlockPuller.cs | 19 ++--- .../Sync/SyncTasks/BlockStartup.cs | 14 ++-- .../Sync/SyncTasks/BlockStore.cs | 34 +++++++-- .../SyncTasks/RichListSync.cs} | 12 +-- .../Sync/SyncTasks/StatsSyncer.cs | 29 +------ .../Storage/Mongo/MongoDataTests.cs | 1 - .../Mongo/MongoStorageOperationsTests.cs | 42 +++++------ src/Blockcore.Indexer/appsettings.json | 5 +- 57 files changed, 350 insertions(+), 429 deletions(-) delete mode 100644 src/Blockcore.Indexer.Cirrus/Storage/Mongo/CirrusStorageBatchFactory.cs delete mode 100644 src/Blockcore.Indexer.Core/Models/BalanceForAddress.cs delete mode 100644 src/Blockcore.Indexer.Core/Models/QueryBlockResults.cs create mode 100644 src/Blockcore.Indexer.Core/Operations/IUtxoCache.cs create mode 100644 src/Blockcore.Indexer.Core/Operations/Types/UtxoCache.cs delete mode 100644 src/Blockcore.Indexer.Core/Storage/DatabaseRegistration.cs rename src/Blockcore.Indexer.Core/Storage/{Mongo => }/IMapMongoBlockToStorageBlock.cs (85%) delete mode 100644 src/Blockcore.Indexer.Core/Storage/IStorageBatchFactory.cs rename src/Blockcore.Indexer.Core/Storage/{Mongo => }/MapMongoBlockToStorageBlock.cs (97%) delete mode 100644 src/Blockcore.Indexer.Core/Storage/Mongo/MongoStorageBatchFactory.cs delete mode 100644 src/Blockcore.Indexer.Core/Storage/Mongo/Types/MongoStorageBatch.cs rename src/Blockcore.Indexer.Core/Storage/{ => Mongo}/Types/Outpoint.cs (81%) delete mode 100644 src/Blockcore.Indexer.Core/Storage/Types/Output.cs delete mode 100644 src/Blockcore.Indexer.Core/Storage/Types/PeerDetails.cs rename src/Blockcore.Indexer.Core/{Storage/Mongo/SyncTasks/MongoDbBlockIndexer.cs => Sync/SyncTasks/BlockIndexer.cs} (96%) rename src/Blockcore.Indexer.Core/{Storage/Mongo/SyncTasks/RichListScanning.cs => Sync/SyncTasks/RichListSync.cs} (95%) diff --git a/README.md b/README.md index 3c01a21a..799fe26d 100644 --- a/README.md +++ b/README.md @@ -66,8 +66,8 @@ The default configurations are read from [appsetting.json](src/Blockcore.Indexer // Syncing parameters "SyncBlockchain": true, "SyncMemoryPool": true, - "DbBatchSize": 10000000, - "DbBatchCount" : 10000, + "MongoBatchSize": 10000000, + "MongoBatchCount" : 10000, "ParallelRequestsToTransactionRpc": 50, "DetailedTrace": 0, "MaxItemsInQueue": 10, @@ -226,4 +226,4 @@ Blockcore.Indexer.Client.BitcoinCommunicationException: Daemon Failed Url = 'htt ---> System.Net.Sockets.SocketException (111): Connection refused ``` -Issue: This happens when the DNS name is not accessible. +Issue: This happens when the DNS name is not accessible. \ No newline at end of file diff --git a/src/Blockcore.Indexer.Angor/Storage/Mongo/Types/Investment.cs b/src/Blockcore.Indexer.Angor/Storage/Mongo/Types/Investment.cs index 23e44732..d82d9ca8 100644 --- a/src/Blockcore.Indexer.Angor/Storage/Mongo/Types/Investment.cs +++ b/src/Blockcore.Indexer.Angor/Storage/Mongo/Types/Investment.cs @@ -1,4 +1,4 @@ -using Blockcore.Indexer.Core.Storage.Types; +using Blockcore.Indexer.Core.Storage.Mongo.Types; namespace Blockcore.Indexer.Angor.Storage.Mongo.Types; diff --git a/src/Blockcore.Indexer.Angor/Sync/SyncTasks/ProjectTransactionsSyncRunner.cs b/src/Blockcore.Indexer.Angor/Sync/SyncTasks/ProjectTransactionsSyncRunner.cs index 4b6a864d..68f8233b 100644 --- a/src/Blockcore.Indexer.Angor/Sync/SyncTasks/ProjectTransactionsSyncRunner.cs +++ b/src/Blockcore.Indexer.Angor/Sync/SyncTasks/ProjectTransactionsSyncRunner.cs @@ -3,7 +3,6 @@ using Blockcore.Indexer.Angor.Storage.Mongo.Types; using Blockcore.Indexer.Core.Settings; using Blockcore.Indexer.Core.Storage.Mongo.Types; -using Blockcore.Indexer.Core.Storage.Types; using Blockcore.Indexer.Core.Sync.SyncTasks; using Blockcore.NBitcoin.DataEncoders; using Microsoft.Extensions.Options; diff --git a/src/Blockcore.Indexer.Angor/Sync/SyncTasks/ProjectsSyncRunner.cs b/src/Blockcore.Indexer.Angor/Sync/SyncTasks/ProjectsSyncRunner.cs index 6becda85..3cd749b8 100644 --- a/src/Blockcore.Indexer.Angor/Sync/SyncTasks/ProjectsSyncRunner.cs +++ b/src/Blockcore.Indexer.Angor/Sync/SyncTasks/ProjectsSyncRunner.cs @@ -6,7 +6,6 @@ using Blockcore.Indexer.Core.Operations.Types; using Blockcore.Indexer.Core.Settings; using Blockcore.Indexer.Core.Storage.Mongo.Types; -using Blockcore.Indexer.Core.Storage.Types; using Blockcore.Indexer.Core.Sync.SyncTasks; using Blockcore.NBitcoin; using Blockcore.NBitcoin.BIP32; diff --git a/src/Blockcore.Indexer.Angor/appsettings.json b/src/Blockcore.Indexer.Angor/appsettings.json index 7ff113db..41b5bdd5 100644 --- a/src/Blockcore.Indexer.Angor/appsettings.json +++ b/src/Blockcore.Indexer.Angor/appsettings.json @@ -18,7 +18,6 @@ } }, "Indexer": { - "DbType": "MongoDb", "ConnectionString": "mongodb://{Symbol}-mongo", "DatabaseNameSubfix": true, @@ -35,8 +34,8 @@ // Syncing parameters "SyncBlockchain": true, "SyncMemoryPool": true, - "DbBatchSize": 10000000, - "DbBatchCount" : 10000, + "MongoBatchSize": 10000000, + "MongoBatchCount" : 10000, "ParallelRequestsToTransactionRpc": 50, "DetailedTrace": 0, "MaxItemsInQueue": 10, diff --git a/src/Blockcore.Indexer.Cirrus/CirrusStartup.cs b/src/Blockcore.Indexer.Cirrus/CirrusStartup.cs index c10ed857..099ec747 100644 --- a/src/Blockcore.Indexer.Cirrus/CirrusStartup.cs +++ b/src/Blockcore.Indexer.Cirrus/CirrusStartup.cs @@ -61,7 +61,6 @@ public void ConfigureServices(IServiceCollection services) services.AddSingleton(); services.AddSingleton(); - services.Replace(new ServiceDescriptor(typeof(IStorageBatchFactory), typeof(CirrusStorageBatchFactory),ServiceLifetime.Singleton)); services.AddControllers() .AddApplicationPart(typeof(Startup).Assembly) diff --git a/src/Blockcore.Indexer.Cirrus/Operations/Types/CirrusStorageBatch.cs b/src/Blockcore.Indexer.Cirrus/Operations/Types/CirrusStorageBatch.cs index 607e54a4..b249d72a 100644 --- a/src/Blockcore.Indexer.Cirrus/Operations/Types/CirrusStorageBatch.cs +++ b/src/Blockcore.Indexer.Cirrus/Operations/Types/CirrusStorageBatch.cs @@ -4,10 +4,11 @@ namespace Blockcore.Indexer.Cirrus.Operations.Types { - public class CirrusStorageBatch : MongoStorageBatch + public class CirrusStorageBatch { - public List CirrusContractTable { get; set; } = new (); + public List CirrusContractTable { get; set; } = new List(); + + public List CirrusContractCodeTable { get; set; } = new List(); - public List CirrusContractCodeTable { get; set; } = new (); } } diff --git a/src/Blockcore.Indexer.Cirrus/Storage/CirrusBlockMapping.cs b/src/Blockcore.Indexer.Cirrus/Storage/CirrusBlockMapping.cs index dd1430aa..aaed4980 100644 --- a/src/Blockcore.Indexer.Cirrus/Storage/CirrusBlockMapping.cs +++ b/src/Blockcore.Indexer.Cirrus/Storage/CirrusBlockMapping.cs @@ -4,7 +4,7 @@ using Blockcore.Indexer.Cirrus.Storage.Mongo.Types; using Blockcore.Indexer.Cirrus.Storage.Types; using Blockcore.Indexer.Core.Client.Types; -using Blockcore.Indexer.Core.Storage.Mongo; +using Blockcore.Indexer.Core.Storage; using Blockcore.Indexer.Core.Storage.Mongo.Types; using Blockcore.Indexer.Core.Storage.Types; using Blockcore.NBitcoin; diff --git a/src/Blockcore.Indexer.Cirrus/Storage/Mongo/CirrusMongoStorageOperations.cs b/src/Blockcore.Indexer.Cirrus/Storage/Mongo/CirrusMongoStorageOperations.cs index 1ad3faf7..bab42737 100644 --- a/src/Blockcore.Indexer.Cirrus/Storage/Mongo/CirrusMongoStorageOperations.cs +++ b/src/Blockcore.Indexer.Cirrus/Storage/Mongo/CirrusMongoStorageOperations.cs @@ -28,6 +28,7 @@ public class CirrusMongoStorageOperations : MongoStorageOperations public CirrusMongoStorageOperations( SyncConnection syncConnection, IStorage storage, + IUtxoCache utxoCache, IOptions configuration, GlobalState globalState, IMapMongoBlockToStorageBlock mongoBlockToStorageBlock, @@ -37,6 +38,7 @@ public CirrusMongoStorageOperations( base( syncConnection, db, + utxoCache, configuration, globalState, mongoBlockToStorageBlock, @@ -50,7 +52,12 @@ public CirrusMongoStorageOperations( protected override void OnAddToStorageBatch(StorageBatch storageBatch, SyncBlockTransactionsOperation item) { - CirrusStorageBatch cirrusStorageBatch = storageBatch as CirrusStorageBatch; + storageBatch.ExtraData ??= new CirrusStorageBatch(); + + if (!(storageBatch.ExtraData is CirrusStorageBatch cirrusStorageBatch)) + { + throw new ArgumentNullException(nameof(cirrusStorageBatch)); + } foreach (Transaction transaction in item.Transactions) { @@ -118,7 +125,10 @@ protected override void OnAddToStorageBatch(StorageBatch storageBatch, SyncBlock protected override void OnPushStorageBatch(StorageBatch storageBatch) { - CirrusStorageBatch cirrusStorageBatch = storageBatch as CirrusStorageBatch; + if (!(storageBatch.ExtraData is CirrusStorageBatch cirrusStorageBatch)) + { + throw new ArgumentNullException(nameof(cirrusStorageBatch)); + } var t1 = Task.Run(() => { diff --git a/src/Blockcore.Indexer.Cirrus/Storage/Mongo/CirrusStorageBatchFactory.cs b/src/Blockcore.Indexer.Cirrus/Storage/Mongo/CirrusStorageBatchFactory.cs deleted file mode 100644 index 15cc8869..00000000 --- a/src/Blockcore.Indexer.Cirrus/Storage/Mongo/CirrusStorageBatchFactory.cs +++ /dev/null @@ -1,10 +0,0 @@ -using Blockcore.Indexer.Cirrus.Operations.Types; -using Blockcore.Indexer.Core.Operations.Types; -using Blockcore.Indexer.Core.Storage; - -namespace Blockcore.Indexer.Cirrus.Storage.Mongo; - -public class CirrusStorageBatchFactory : IStorageBatchFactory -{ - public StorageBatch GetStorageBatch() => new CirrusStorageBatch(); -} diff --git a/src/Blockcore.Indexer.Cirrus/appsettings.json b/src/Blockcore.Indexer.Cirrus/appsettings.json index be7b749c..7e6e450b 100644 --- a/src/Blockcore.Indexer.Cirrus/appsettings.json +++ b/src/Blockcore.Indexer.Cirrus/appsettings.json @@ -18,7 +18,7 @@ } }, "Indexer": { - "DbType": "MongoDb", + // "ConnectionString": "mongodb://localhost", "ConnectionString": "mongodb://{Symbol}-mongo", "DatabaseNameSubfix": true, @@ -34,8 +34,8 @@ // Syncing parameters "SyncBlockchain": true, "SyncMemoryPool": true, - "DbBatchSize": 10000000, - "DbBatchCount" : 10000, + "MongoBatchSize": 10000000, + "MongoBatchCount" : 10000, "ParallelRequestsToTransactionRpc": 50, "DetailedTrace": 0, "MaxItemsInQueue": 10, diff --git a/src/Blockcore.Indexer.Core/Client/Types/PeerInfo.cs b/src/Blockcore.Indexer.Core/Client/Types/PeerInfo.cs index 09954cc7..dadd19a8 100644 --- a/src/Blockcore.Indexer.Core/Client/Types/PeerInfo.cs +++ b/src/Blockcore.Indexer.Core/Client/Types/PeerInfo.cs @@ -1,9 +1,11 @@ using System; using System.Collections.Generic; +using MongoDB.Bson.Serialization.Attributes; using Newtonsoft.Json; namespace Blockcore.Indexer.Core.Client.Types { + [BsonIgnoreExtraElements] public class PeerInfo { public string Addr { get; set; } diff --git a/src/Blockcore.Indexer.Core/Controllers/InsightController.cs b/src/Blockcore.Indexer.Core/Controllers/InsightController.cs index aa224ae8..4eaf7872 100644 --- a/src/Blockcore.Indexer.Core/Controllers/InsightController.cs +++ b/src/Blockcore.Indexer.Core/Controllers/InsightController.cs @@ -172,7 +172,7 @@ private Supply CalculateSupply() /// private List RetrieveWallets() { - // TODO: Funds should be stored in the DB, and be editable by individual chains and not hard-coded. + // TODO: Funds should be stored in MongoDB, and be editable by individual chains and not hard-coded. var funds = new List(); List wallets = insightConfiguration.Wallets; @@ -181,7 +181,7 @@ private List RetrieveWallets() { if (wallet.Address != null && wallet.Address.Length > 0) { - var balances = storage.AddressBalances(wallet.Address); + List balances = storage.AddressBalances(wallet.Address); long balance = balances.Sum(b => b.Balance); wallet.Balance = balance; } @@ -207,7 +207,7 @@ private decimal CalculateCirculatingSupply() } } - var balances = storage.AddressBalances(listOfAddress); + List balances = storage.AddressBalances(listOfAddress); long walletBalances = balances.Sum(b => b.Balance); long circulatingSupply = totalBalance - walletBalances; diff --git a/src/Blockcore.Indexer.Core/Controllers/QueryController.cs b/src/Blockcore.Indexer.Core/Controllers/QueryController.cs index 64f7a3fe..44ceabf3 100644 --- a/src/Blockcore.Indexer.Core/Controllers/QueryController.cs +++ b/src/Blockcore.Indexer.Core/Controllers/QueryController.cs @@ -79,7 +79,7 @@ public IActionResult GetAddressTransactions([MinLength(4)][MaxLength(100)] strin [Route("address/{address}/transactions/unspent")] public async Task GetAddressTransactionsUnspent([MinLength(30)][MaxLength(100)] string address, long confirmations = 0, [Range(0, int.MaxValue)] int offset = 0, [Range(1, 50)] int limit = 10) { - var result = await storage.GetUnspentTransactionsByAddressAsync(address, confirmations, offset, limit); + QueryResult result = await storage.GetUnspentTransactionsByAddressAsync(address, confirmations, offset, limit); return OkPaging(result); } diff --git a/src/Blockcore.Indexer.Core/Controllers/StatsController.cs b/src/Blockcore.Indexer.Core/Controllers/StatsController.cs index 43813688..4b5b2b13 100644 --- a/src/Blockcore.Indexer.Core/Controllers/StatsController.cs +++ b/src/Blockcore.Indexer.Core/Controllers/StatsController.cs @@ -1,11 +1,10 @@ using System; using System.Collections.Generic; -using System.Linq; using System.Threading.Tasks; -using Blockcore.Indexer.Core.Client.Types; using Blockcore.Indexer.Core.Handlers; using Blockcore.Indexer.Core.Models; using Blockcore.Indexer.Core.Storage; +using Blockcore.Indexer.Core.Storage.Mongo; using Microsoft.AspNetCore.Mvc; namespace Blockcore.Indexer.Core.Controllers @@ -19,7 +18,7 @@ public class StatsController : ControllerBase { private readonly StatsHandler statsHandler; - private readonly IStorage storage; + private readonly MongoData storage; /// /// Initializes a new instance of the class. @@ -27,7 +26,7 @@ public class StatsController : ControllerBase public StatsController(StatsHandler statsHandler, IStorage storage) { this.statsHandler = statsHandler; - this.storage = storage; + this.storage = storage as MongoData; } [HttpGet] @@ -84,9 +83,7 @@ public async Task Peers() [Route("peers/{date}")] public IActionResult Peers(DateTime date) { - List list = storage.GetPeerFromDate(date) - .Select(x => x as PeerInfo) - .ToList(); + List list = storage.GetPeerFromDate(date); return Ok(list); } diff --git a/src/Blockcore.Indexer.Core/Models/BalanceForAddress.cs b/src/Blockcore.Indexer.Core/Models/BalanceForAddress.cs deleted file mode 100644 index 53191459..00000000 --- a/src/Blockcore.Indexer.Core/Models/BalanceForAddress.cs +++ /dev/null @@ -1,7 +0,0 @@ -namespace Blockcore.Indexer.Core.Models; - -public class BalanceForAddress -{ - public string Address { get; set; } - public long Balance { get; set; } -} diff --git a/src/Blockcore.Indexer.Core/Models/QueryBlock.cs b/src/Blockcore.Indexer.Core/Models/QueryBlock.cs index b48136d5..8c50a81d 100644 --- a/src/Blockcore.Indexer.Core/Models/QueryBlock.cs +++ b/src/Blockcore.Indexer.Core/Models/QueryBlock.cs @@ -1,9 +1,16 @@ +using System.Collections.Generic; namespace Blockcore.Indexer.Core.Models { - public class QueryBlock + public class QueryBlockResults { + public IEnumerable Blocks { get; set; } + + public int Total { get; set; } + } + public class QueryBlock + { /// /// Gets or sets the Symbol. /// @@ -53,16 +60,16 @@ public class QueryBlock public string Bits { get; set; } - public double Difficulty { get; set; } - - public string ChainWork { get; set; } - public string Merkleroot { get; set; } public long Nonce { get; set; } public long Version { get; set; } + public double Difficulty { get; set; } + + public string ChainWork { get; set; } + public string PosBlockSignature { get; set; } public string PosModifierv2 { get; set; } @@ -74,5 +81,10 @@ public class QueryBlock public string PosBlockTrust { get; set; } public string PosChainTrust { get; set; } + + /// + /// Gets or sets the transactions. + /// + public IEnumerable Transactions { get; set; } } } diff --git a/src/Blockcore.Indexer.Core/Models/QueryBlockResults.cs b/src/Blockcore.Indexer.Core/Models/QueryBlockResults.cs deleted file mode 100644 index 2935610f..00000000 --- a/src/Blockcore.Indexer.Core/Models/QueryBlockResults.cs +++ /dev/null @@ -1,10 +0,0 @@ -using System.Collections.Generic; - -namespace Blockcore.Indexer.Core.Models; - -public class QueryBlockResults -{ - public IEnumerable Blocks { get; set; } - - public int Total { get; set; } -} diff --git a/src/Blockcore.Indexer.Core/Models/QueryOrphanBlock.cs b/src/Blockcore.Indexer.Core/Models/QueryOrphanBlock.cs index dc8019b1..502986fe 100644 --- a/src/Blockcore.Indexer.Core/Models/QueryOrphanBlock.cs +++ b/src/Blockcore.Indexer.Core/Models/QueryOrphanBlock.cs @@ -1,5 +1,6 @@ using System; -using Blockcore.Indexer.Core.Storage.Types; +using System.Collections.Generic; +using Blockcore.Indexer.Core.Storage.Mongo.Types; namespace Blockcore.Indexer.Core.Models { @@ -8,6 +9,6 @@ public class QueryOrphanBlock public DateTime Created { get; set; } public uint BlockIndex { get; set; } public string BlockHash { get; set; } - public QueryBlock Block { get; set; } + public BlockTable Block { get; set; } } } diff --git a/src/Blockcore.Indexer.Core/Operations/IStorageOperations.cs b/src/Blockcore.Indexer.Core/Operations/IStorageOperations.cs index 01a4c055..ad437454 100644 --- a/src/Blockcore.Indexer.Core/Operations/IStorageOperations.cs +++ b/src/Blockcore.Indexer.Core/Operations/IStorageOperations.cs @@ -13,6 +13,6 @@ public interface IStorageOperations SyncBlockInfo PushStorageBatch(StorageBatch storageBatch); - void InsertMempoolTransactions(SyncBlockTransactionsOperation item); + InsertStats InsertMempoolTransactions(SyncBlockTransactionsOperation item); } } diff --git a/src/Blockcore.Indexer.Core/Operations/IUtxoCache.cs b/src/Blockcore.Indexer.Core/Operations/IUtxoCache.cs new file mode 100644 index 00000000..57a5089f --- /dev/null +++ b/src/Blockcore.Indexer.Core/Operations/IUtxoCache.cs @@ -0,0 +1,19 @@ +using System.Collections.Generic; +using Blockcore.Indexer.Core.Operations.Types; +using Blockcore.Indexer.Core.Storage.Mongo.Types; + +namespace Blockcore.Indexer.Core.Operations +{ + /// + /// Maintain a cache of unspent outputs + /// + public interface IUtxoCache + { + int CacheSize { get; } + + UtxoCacheItem GetOne(string outpoint); + void AddToCache(IEnumerable outputs); + + void RemoveFromCache(IEnumerable inputs); + } +} diff --git a/src/Blockcore.Indexer.Core/Operations/Types/InsertStats.cs b/src/Blockcore.Indexer.Core/Operations/Types/InsertStats.cs index ad148d41..0631a30b 100644 --- a/src/Blockcore.Indexer.Core/Operations/Types/InsertStats.cs +++ b/src/Blockcore.Indexer.Core/Operations/Types/InsertStats.cs @@ -1,3 +1,6 @@ +using System.Collections.Generic; +using Blockcore.Indexer.Core.Storage.Mongo.Types; + namespace Blockcore.Indexer.Core.Operations.Types { #region Using Directives @@ -20,5 +23,10 @@ public class InsertStats /// Gets or sets the outputs. /// public int InputsOutputs { get; set; } + + /// + /// Gets or sets the items. + /// + public List Items { get; set; } } } diff --git a/src/Blockcore.Indexer.Core/Operations/Types/StorageBatch.cs b/src/Blockcore.Indexer.Core/Operations/Types/StorageBatch.cs index 057866d4..f6a5084e 100644 --- a/src/Blockcore.Indexer.Core/Operations/Types/StorageBatch.cs +++ b/src/Blockcore.Indexer.Core/Operations/Types/StorageBatch.cs @@ -1,4 +1,5 @@ using System.Collections.Generic; +using Blockcore.Indexer.Core.Storage.Mongo.Types; namespace Blockcore.Indexer.Core.Operations.Types { @@ -6,15 +7,15 @@ namespace Blockcore.Indexer.Core.Operations.Types #endregion Using Directives - public abstract class StorageBatch + public class StorageBatch { - public abstract int GetBlockCount(); - public abstract int GetOutputCount(); - public abstract int GetInputCount(); - public abstract int GetTransactionCount(); - public abstract long GetBatchSize(); + public long TotalSize { get; set; } + public List TransactionBlockTable { get; set; } = new(); + public Dictionary BlockTable { get; set; } = new(); + public List TransactionTable { get; set; } = new(); + public Dictionary OutputTable { get; set; } = new(); + public List InputTable { get; set; } = new(); - public abstract IEnumerable GetBlockSizes(); - public abstract bool ValidateBatch(string prevBlockHash); + public object ExtraData { get; set; } } } diff --git a/src/Blockcore.Indexer.Core/Operations/Types/UtxoCache.cs b/src/Blockcore.Indexer.Core/Operations/Types/UtxoCache.cs new file mode 100644 index 00000000..f4636bc6 --- /dev/null +++ b/src/Blockcore.Indexer.Core/Operations/Types/UtxoCache.cs @@ -0,0 +1,57 @@ +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using Blockcore.Indexer.Core.Storage.Mongo.Types; +using Microsoft.Extensions.Logging; + +namespace Blockcore.Indexer.Core.Operations.Types +{ + public class UtxoCacheItem + { + public string Address { get; set; } + public long Value { get; set; } + } + + public class UtxoCache : IUtxoCache + { + private readonly ILogger logger; + private readonly ConcurrentDictionary cache; + + private readonly int maxItemInCache = 0; + + public UtxoCache(ILogger logger) + { + this.logger = logger; + cache = new ConcurrentDictionary(); + } + + public int CacheSize { get { return cache.Count; } } + + public UtxoCacheItem GetOne(string outpoint) + { + if (cache.TryGetValue(outpoint, out UtxoCacheItem utxo)) + { + return new UtxoCacheItem { Value = utxo.Value, Address = utxo.Address }; + } + + return null; + } + + public void AddToCache(IEnumerable outputs) + { + int maxToAdd = maxItemInCache - cache.Count; + foreach (OutputTable output in outputs.Take(maxToAdd)) + { + cache.TryAdd($"{output.Outpoint.TransactionId}-{output.Outpoint.OutputIndex}", new UtxoCacheItem { Value = output.Value, Address = output.Address }); + } + } + + public void RemoveFromCache(IEnumerable inputs) + { + foreach (InputTable output in inputs) + { + cache.TryRemove($"{output.Outpoint.TransactionId}-{output.Outpoint.OutputIndex}", out _); + } + } + } +} diff --git a/src/Blockcore.Indexer.Core/Settings/IndexerSettings.cs b/src/Blockcore.Indexer.Core/Settings/IndexerSettings.cs index e511f1ad..6d4eea37 100644 --- a/src/Blockcore.Indexer.Core/Settings/IndexerSettings.cs +++ b/src/Blockcore.Indexer.Core/Settings/IndexerSettings.cs @@ -32,8 +32,8 @@ public class IndexerSettings public int NotifyBatchCount { get; set; } - public int DbBatchSize { get; set; } - public int DbBatchCount { get; set; } + public int MongoBatchSize { get; set; } + public int MongoBatchCount { get; set; } public int AverageInterval { get; set; } @@ -44,7 +44,5 @@ public class IndexerSettings public int MaxItemsInHistoryQueue { get; set; } public int IndexCountForBlockIndexProperty { get;set; } - - public string DbType { get; set; } } } diff --git a/src/Blockcore.Indexer.Core/Startup.cs b/src/Blockcore.Indexer.Core/Startup.cs index 3fcc0a8c..2e58db94 100644 --- a/src/Blockcore.Indexer.Core/Startup.cs +++ b/src/Blockcore.Indexer.Core/Startup.cs @@ -15,13 +15,17 @@ using Blockcore.Indexer.Core.Storage.Mongo; using Blockcore.Indexer.Core.Sync; using Blockcore.Indexer.Core.Sync.SyncTasks; +using Blockcore.Utilities; +using ConcurrentCollections; using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Hosting; using Microsoft.AspNetCore.Mvc.ApplicationModels; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; using Microsoft.Extensions.PlatformAbstractions; using Microsoft.OpenApi.Models; +using MongoDB.Driver; using Newtonsoft.Json; namespace Blockcore.Indexer.Core @@ -35,22 +39,30 @@ public static void AddIndexerServices(IServiceCollection services, IConfiguratio services.Configure(configuration.GetSection("Indexer")); services.Configure(configuration.GetSection("Insight")); - IndexerSettings indexer = new IndexerSettings(); - configuration.GetSection("Indexer").Bind(indexer); - - switch (indexer.DbType) + services.AddSingleton(_ => { - case "MongoDb": - services.AddMongoDatabase(); - break; - default: throw new InvalidOperationException(); - } + var indexerConfiguration = _.GetService(typeof(IOptions))as IOptions ;// configuration.GetSection("Indexer") as IndexerSettings; + var chainConfiguration = _.GetService(typeof(IOptions)) as IOptions;// configuration.GetSection("Chain") as ChainSettings; + + var mongoClient = new MongoClient(indexerConfiguration.Value.ConnectionString.Replace("{Symbol}", + chainConfiguration.Value.Symbol.ToLower())); + + string dbName = indexerConfiguration.Value.DatabaseNameSubfix + ? $"Blockchain{chainConfiguration.Value.Symbol}" + : "Blockchain"; + + return mongoClient.GetDatabase(dbName); + }); // services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); - + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); services.AddTransient(); services.AddSingleton(); services.AddSingleton(); @@ -69,6 +81,9 @@ public static void AddIndexerServices(IServiceCollection services, IConfiguratio services.AddScoped(); services.AddScoped(); + services.AddScoped(); + services.AddScoped(); + // TODO: Verify that it is OK we add this to shared Startup for Blockcore and Cirrus. services.AddScoped(); services.AddSingleton(); @@ -127,7 +142,7 @@ public static void AddIndexerServices(IServiceCollection services, IConfiguratio .AllowAnyHeader(); })); - + services.AddTransient(); services.AddSingleton(); services.AddSingleton(); diff --git a/src/Blockcore.Indexer.Core/Storage/DatabaseRegistration.cs b/src/Blockcore.Indexer.Core/Storage/DatabaseRegistration.cs deleted file mode 100644 index d9337fc0..00000000 --- a/src/Blockcore.Indexer.Core/Storage/DatabaseRegistration.cs +++ /dev/null @@ -1,43 +0,0 @@ -using Blockcore.Indexer.Core.Operations; -using Blockcore.Indexer.Core.Settings; -using Blockcore.Indexer.Core.Storage.Mongo; -using Blockcore.Indexer.Core.Storage.Mongo.SyncTasks; -using Blockcore.Indexer.Core.Sync.SyncTasks; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Options; -using MongoDB.Driver; - -namespace Blockcore.Indexer.Core.Storage; - -public static class DatabaseRegistration -{ - public static IServiceCollection AddMongoDatabase( this IServiceCollection services) - { - services.AddSingleton(); - services.AddSingleton(); - services.AddSingleton(); - services.AddSingleton(); - services.AddTransient(); - services.AddScoped(); - services.AddSingleton(); - //TODO add this for address driven blockchains - //services.AddScoped(); - - services.AddSingleton(_ => - { - var indexerConfiguration = _.GetService(typeof(IOptions))as IOptions ;// configuration.GetSection("Indexer") as IndexerSettings; - var chainConfiguration = _.GetService(typeof(IOptions)) as IOptions;// configuration.GetSection("Chain") as ChainSettings; - - var mongoClient = new MongoClient(indexerConfiguration.Value.ConnectionString.Replace("{Symbol}", - chainConfiguration.Value.Symbol.ToLower())); - - string dbName = indexerConfiguration.Value.DatabaseNameSubfix - ? $"Blockchain{chainConfiguration.Value.Symbol}" - : "Blockchain"; - - return mongoClient.GetDatabase(dbName); - }); - - return services; - } -} diff --git a/src/Blockcore.Indexer.Core/Storage/Mongo/IMapMongoBlockToStorageBlock.cs b/src/Blockcore.Indexer.Core/Storage/IMapMongoBlockToStorageBlock.cs similarity index 85% rename from src/Blockcore.Indexer.Core/Storage/Mongo/IMapMongoBlockToStorageBlock.cs rename to src/Blockcore.Indexer.Core/Storage/IMapMongoBlockToStorageBlock.cs index b1060a39..8d93a850 100644 --- a/src/Blockcore.Indexer.Core/Storage/Mongo/IMapMongoBlockToStorageBlock.cs +++ b/src/Blockcore.Indexer.Core/Storage/IMapMongoBlockToStorageBlock.cs @@ -2,7 +2,7 @@ using Blockcore.Indexer.Core.Storage.Mongo.Types; using Blockcore.Indexer.Core.Storage.Types; -namespace Blockcore.Indexer.Core.Storage.Mongo +namespace Blockcore.Indexer.Core.Storage { public interface IMapMongoBlockToStorageBlock { diff --git a/src/Blockcore.Indexer.Core/Storage/IStorage.cs b/src/Blockcore.Indexer.Core/Storage/IStorage.cs index 319a3b9f..3f21e375 100644 --- a/src/Blockcore.Indexer.Core/Storage/IStorage.cs +++ b/src/Blockcore.Indexer.Core/Storage/IStorage.cs @@ -1,4 +1,3 @@ -using System; using System.Collections.Generic; using System.Threading.Tasks; using Blockcore.Indexer.Core.Models; @@ -44,9 +43,11 @@ Task> QuickBalancesLookupForAddressesWithHistoryCheckA ReorgBlockTable OrphanBlockByHash(string blockHash); - QueryResult Richlist(int offset, int limit); + QueryResult Richlist(int offset, int limit); - List AddressBalances(IEnumerable addresses); + RichlistTable RichlistBalance(string address); + + List AddressBalances(IEnumerable addresses); long TotalBalance(); @@ -57,12 +58,5 @@ Task> QuickBalancesLookupForAddressesWithHistoryCheckA public List GetIndexesBuildProgress(); public List GetBlockIndexIndexes(); - - public List GetMempoolTransactionIds(); - - public bool DeleteTransactionsFromMempool(List transactionIds); - - List GetPeerFromDate(DateTime date); - Task InsertPeer(PeerDetails info); } } diff --git a/src/Blockcore.Indexer.Core/Storage/IStorageBatchFactory.cs b/src/Blockcore.Indexer.Core/Storage/IStorageBatchFactory.cs deleted file mode 100644 index 106aed35..00000000 --- a/src/Blockcore.Indexer.Core/Storage/IStorageBatchFactory.cs +++ /dev/null @@ -1,8 +0,0 @@ -using Blockcore.Indexer.Core.Operations.Types; - -namespace Blockcore.Indexer.Core.Storage; - -public interface IStorageBatchFactory -{ - StorageBatch GetStorageBatch(); -} diff --git a/src/Blockcore.Indexer.Core/Storage/Mongo/MapMongoBlockToStorageBlock.cs b/src/Blockcore.Indexer.Core/Storage/MapMongoBlockToStorageBlock.cs similarity index 97% rename from src/Blockcore.Indexer.Core/Storage/Mongo/MapMongoBlockToStorageBlock.cs rename to src/Blockcore.Indexer.Core/Storage/MapMongoBlockToStorageBlock.cs index 6a4b2b6f..a4604ada 100644 --- a/src/Blockcore.Indexer.Core/Storage/Mongo/MapMongoBlockToStorageBlock.cs +++ b/src/Blockcore.Indexer.Core/Storage/MapMongoBlockToStorageBlock.cs @@ -3,7 +3,7 @@ using Blockcore.Indexer.Core.Storage.Mongo.Types; using Blockcore.Indexer.Core.Storage.Types; -namespace Blockcore.Indexer.Core.Storage.Mongo +namespace Blockcore.Indexer.Core.Storage { public class MapMongoBlockToStorageBlock : IMapMongoBlockToStorageBlock { diff --git a/src/Blockcore.Indexer.Core/Storage/Mongo/BlockRewindOperation.cs b/src/Blockcore.Indexer.Core/Storage/Mongo/BlockRewindOperation.cs index 74fe76c4..a3636188 100644 --- a/src/Blockcore.Indexer.Core/Storage/Mongo/BlockRewindOperation.cs +++ b/src/Blockcore.Indexer.Core/Storage/Mongo/BlockRewindOperation.cs @@ -4,7 +4,6 @@ using System.Linq.Expressions; using System.Threading.Tasks; using Blockcore.Indexer.Core.Storage.Mongo.Types; -using Blockcore.Indexer.Core.Storage.Types; using MongoDB.Bson; using MongoDB.Driver; diff --git a/src/Blockcore.Indexer.Core/Storage/Mongo/IMongoDb.cs b/src/Blockcore.Indexer.Core/Storage/Mongo/IMongoDb.cs index b11dfc92..feb49a64 100644 --- a/src/Blockcore.Indexer.Core/Storage/Mongo/IMongoDb.cs +++ b/src/Blockcore.Indexer.Core/Storage/Mongo/IMongoDb.cs @@ -1,6 +1,5 @@ using Blockcore.Indexer.Core.Client.Types; using Blockcore.Indexer.Core.Storage.Mongo.Types; -using Blockcore.Indexer.Core.Storage.Types; using MongoDB.Driver; namespace Blockcore.Indexer.Core.Storage.Mongo; @@ -16,7 +15,7 @@ public interface IMongoDb IMongoCollection TransactionTable { get; } IMongoCollection BlockTable { get; } IMongoCollection RichlistTable { get; } - IMongoCollection Peer { get; } + IMongoCollection Peer { get; } IMongoCollection Mempool { get; } IMongoCollection ReorgBlock { get; } } diff --git a/src/Blockcore.Indexer.Core/Storage/Mongo/MongoBuilder.cs b/src/Blockcore.Indexer.Core/Storage/Mongo/MongoBuilder.cs index 32f89829..286777de 100644 --- a/src/Blockcore.Indexer.Core/Storage/Mongo/MongoBuilder.cs +++ b/src/Blockcore.Indexer.Core/Storage/Mongo/MongoBuilder.cs @@ -1,7 +1,6 @@ using System.Threading.Tasks; using Blockcore.Indexer.Core.Settings; using Blockcore.Indexer.Core.Storage.Mongo.Types; -using Blockcore.Indexer.Core.Storage.Types; using Blockcore.Indexer.Core.Sync.SyncTasks; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; @@ -144,15 +143,6 @@ public override Task OnExecute() }); } - if (!MongoDB.Bson.Serialization.BsonClassMap.IsClassMapRegistered(typeof(PeerDetails))) - { - MongoDB.Bson.Serialization.BsonClassMap.RegisterClassMap(cm => - { - cm.AutoMap(); - cm.SetIgnoreExtraElements(true); - }); - } - mongoDb.UnspentOutputTable.Indexes .CreateOne(new CreateIndexModel(Builders .IndexKeys.Hashed(trxBlk => trxBlk.Outpoint))); diff --git a/src/Blockcore.Indexer.Core/Storage/Mongo/MongoData.cs b/src/Blockcore.Indexer.Core/Storage/Mongo/MongoData.cs index 165189f3..4fd8c3e7 100644 --- a/src/Blockcore.Indexer.Core/Storage/Mongo/MongoData.cs +++ b/src/Blockcore.Indexer.Core/Storage/Mongo/MongoData.cs @@ -5,7 +5,9 @@ using Blockcore.Consensus.ScriptInfo; using Blockcore.Consensus.TransactionInfo; using Blockcore.Indexer.Core.Client; +using Blockcore.Indexer.Core.Client.Types; using Blockcore.Indexer.Core.Crypto; +using Blockcore.Indexer.Core.Extensions; using Blockcore.Indexer.Core.Models; using Blockcore.Indexer.Core.Operations.Types; using Blockcore.Indexer.Core.Settings; @@ -17,6 +19,7 @@ using MongoDB.Bson; using MongoDB.Driver; using Blockcore.NBitcoin.DataEncoders; +using Blockcore.Utilities; namespace Blockcore.Indexer.Core.Storage.Mongo { @@ -80,18 +83,6 @@ public List GetBlockIndexIndexes() return indexNames.Where(w => w.Contains("BlockIndex")).ToList(); } - public List GetMempoolTransactionIds() => mongoDb.Mempool.AsQueryable().Select(x => x.TransactionId).ToList(); - - public bool DeleteTransactionsFromMempool(List transactionIds) - { - FilterDefinitionBuilder builder = Builders.Filter; - FilterDefinition filter = builder.In(mempoolItem => mempoolItem.TransactionId, transactionIds); - - var result = mongoDb.Mempool.DeleteMany(filter); - - return result.IsAcknowledged; //TODO should we change this to count == count of transaction ids? - } - public List GetIndexesBuildProgress() { IMongoDatabase db = mongoDatabase.Client.GetDatabase("admin"); @@ -281,7 +272,7 @@ public QueryResult OrphanBlocks(int? offset, int limit) BlockHash = s.BlockHash, BlockIndex = s.BlockIndex, Created = s.Created, - Block = MapQueryBlock(s.Block) + Block = s.Block }), Total = total, Offset = itemsToSkip, @@ -289,34 +280,6 @@ public QueryResult OrphanBlocks(int? offset, int limit) }; } - private static QueryBlock MapQueryBlock(BlockTable blockTable) - { - return new QueryBlock() - { - BlockHash = blockTable.BlockHash, - BlockIndex = blockTable.BlockIndex, - BlockSize = blockTable.BlockSize, - BlockTime = blockTable.BlockTime, - NextBlockHash = blockTable.NextBlockHash, - PreviousBlockHash = blockTable.PreviousBlockHash, - Confirmations = blockTable.Confirmations, - Bits = blockTable.Bits, - Difficulty = blockTable.Difficulty, - ChainWork = blockTable.ChainWork, - Merkleroot = blockTable.Merkleroot, - Nonce = blockTable.Nonce, - Version = blockTable.Version, - Synced = blockTable.SyncComplete, - TransactionCount = blockTable.TransactionCount, - PosBlockSignature = blockTable.PosBlockSignature, - PosModifierv2 = blockTable.PosModifierv2, - PosFlags = blockTable.PosFlags, - PosHashProof = blockTable.PosHashProof, - PosBlockTrust = blockTable.PosBlockTrust, - PosChainTrust = blockTable.PosChainTrust, - }; - } - public ReorgBlockTable OrphanBlockByHash(string blockHash) { FilterDefinition filter = Builders.Filter.Eq(info => info.BlockHash, blockHash); @@ -329,7 +292,7 @@ public ReorgBlockTable OrphanBlockByHash(string blockHash) /// /// /// - public async Task InsertPeer(PeerDetails info) + public async Task InsertPeer(PeerInfo info) { // Always update the LastSeen. info.LastSeen = DateTime.UtcNow; @@ -339,9 +302,9 @@ public async Task InsertPeer(PeerDetails info) return replaceOneResult.ModifiedCount; } - public List GetPeerFromDate(DateTime date) + public List GetPeerFromDate(DateTime date) { - FilterDefinition filter = Builders.Filter.Gt(addr => addr.LastSeen, date); + FilterDefinition filter = Builders.Filter.Gt(addr => addr.LastSeen, date); return mongoDb.Peer.Find(filter).ToList(); } @@ -511,7 +474,7 @@ public SyncTransactionItems TransactionItemsGet(string transactionId, Transactio return ret; } - public QueryResult Richlist(int offset, int limit) + public QueryResult Richlist(int offset, int limit) { FilterDefinitionBuilder filterBuilder = Builders.Filter; FilterDefinition filter = filterBuilder.Empty; @@ -539,19 +502,27 @@ public QueryResult Richlist(int offset, int limit) .Limit(limit) .ToList(); - return new QueryResult { Items = list.Select(x => new BalanceForAddress - { Address = x.Address,Balance = x.Balance }), Total = total, Offset = offset, Limit = limit }; + return new QueryResult { Items = list, Total = total, Offset = offset, Limit = limit }; + } + + public RichlistTable RichlistBalance(string address) + { + FilterDefinitionBuilder filterBuilder = Builders.Filter; + FilterDefinition filter = filterBuilder.Eq(m => m.Address, address); + + RichlistTable table = mongoDb.RichlistTable.Find(filter).SingleOrDefault(); + + return table; } - public List AddressBalances(IEnumerable addresses) + public List AddressBalances(IEnumerable addresses) { FilterDefinitionBuilder filterBuilder = Builders.Filter; FilterDefinition filter = filterBuilder.Where(s => addresses.Contains(s.Address)); - return mongoDb.RichlistTable.Find(filter) - .ToList() - .Select(x => new BalanceForAddress { Balance = x.Balance, Address = x.Address }) - .ToList(); + List document = mongoDb.RichlistTable.Find(filter).ToList(); + + return document; } public long TotalBalance() diff --git a/src/Blockcore.Indexer.Core/Storage/Mongo/MongoDb.cs b/src/Blockcore.Indexer.Core/Storage/Mongo/MongoDb.cs index a27d5d5b..c7e02fda 100644 --- a/src/Blockcore.Indexer.Core/Storage/Mongo/MongoDb.cs +++ b/src/Blockcore.Indexer.Core/Storage/Mongo/MongoDb.cs @@ -1,7 +1,9 @@ using Blockcore.Indexer.Core.Client.Types; +using Blockcore.Indexer.Core.Operations.Types; +using Blockcore.Indexer.Core.Settings; using Blockcore.Indexer.Core.Storage.Mongo.Types; -using Blockcore.Indexer.Core.Storage.Types; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; using MongoDB.Driver; namespace Blockcore.Indexer.Core.Storage.Mongo @@ -91,11 +93,11 @@ public IMongoCollection RichlistTable } } - public IMongoCollection Peer + public IMongoCollection Peer { get { - return mongoDatabase.GetCollection("Peer"); + return mongoDatabase.GetCollection("Peer"); } } diff --git a/src/Blockcore.Indexer.Core/Storage/Mongo/MongoStorageBatchFactory.cs b/src/Blockcore.Indexer.Core/Storage/Mongo/MongoStorageBatchFactory.cs deleted file mode 100644 index 27046920..00000000 --- a/src/Blockcore.Indexer.Core/Storage/Mongo/MongoStorageBatchFactory.cs +++ /dev/null @@ -1,9 +0,0 @@ -using Blockcore.Indexer.Core.Operations.Types; -using Blockcore.Indexer.Core.Storage.Mongo.Types; - -namespace Blockcore.Indexer.Core.Storage.Mongo; - -public class MongoStorageBatchFactory : IStorageBatchFactory -{ - public StorageBatch GetStorageBatch() => new MongoStorageBatch(); -} diff --git a/src/Blockcore.Indexer.Core/Storage/Mongo/MongoStorageOperations.cs b/src/Blockcore.Indexer.Core/Storage/Mongo/MongoStorageOperations.cs index 2c758c9a..0891f3da 100644 --- a/src/Blockcore.Indexer.Core/Storage/Mongo/MongoStorageOperations.cs +++ b/src/Blockcore.Indexer.Core/Storage/Mongo/MongoStorageOperations.cs @@ -30,6 +30,7 @@ public class MongoStorageOperations : IStorageOperations public MongoStorageOperations( SyncConnection syncConnection, IMongoDb storage, + IUtxoCache utxoCache, IOptions configuration, GlobalState globalState, IMapMongoBlockToStorageBlock mongoBlockToStorageBlock, @@ -47,17 +48,15 @@ public MongoStorageOperations( public void AddToStorageBatch(StorageBatch storageBatch, SyncBlockTransactionsOperation item) { - var mongoStorageBatch = storageBatch as MongoStorageBatch; - - mongoStorageBatch.TotalSize += item.BlockInfo.Size; - mongoStorageBatch.BlockTable.Add(item.BlockInfo.Height, mongoBlockToStorageBlock.Map(item.BlockInfo)); + storageBatch.TotalSize += item.BlockInfo.Size; + storageBatch.BlockTable.Add(item.BlockInfo.Height, mongoBlockToStorageBlock.Map(item.BlockInfo)); int transactionIndex = 0; foreach (Transaction trx in item.Transactions) { string trxHash = trx.GetHash().ToString(); - mongoStorageBatch.TransactionBlockTable.Add( + storageBatch.TransactionBlockTable.Add( new TransactionBlockTable { BlockIndex = item.BlockInfo.HeightAsUint32, @@ -68,7 +67,7 @@ public void AddToStorageBatch(StorageBatch storageBatch, SyncBlockTransactionsOp if (configuration.StoreRawTransactions) { - mongoStorageBatch.TransactionTable.Add(new TransactionTable + storageBatch.TransactionTable.Add(new TransactionTable { TransactionId = trxHash, RawTransaction = trx.ToBytes(syncConnection.Network.Consensus.ConsensusFactory) @@ -85,7 +84,7 @@ public void AddToStorageBatch(StorageBatch storageBatch, SyncBlockTransactionsOp var outpoint = new Outpoint { TransactionId = trxHash, OutputIndex = outputIndex++ }; - mongoStorageBatch.OutputTable.Add(outpoint.ToString(), new OutputTable + storageBatch.OutputTable.Add(outpoint.ToString(), new OutputTable { Address = addr, Outpoint = outpoint, @@ -107,9 +106,9 @@ public void AddToStorageBatch(StorageBatch storageBatch, SyncBlockTransactionsOp TransactionId = input.PrevOut.Hash.ToString(), OutputIndex = (int)input.PrevOut.N }; - mongoStorageBatch.OutputTable.TryGetValue(outpoint.ToString(), out OutputTable output); + storageBatch.OutputTable.TryGetValue(outpoint.ToString(), out OutputTable output); - mongoStorageBatch.InputTable.Add(new InputTable() + storageBatch.InputTable.Add(new InputTable() { Outpoint = outpoint, TrxHash = trxHash, @@ -127,15 +126,13 @@ public void AddToStorageBatch(StorageBatch storageBatch, SyncBlockTransactionsOp public SyncBlockInfo PushStorageBatch(StorageBatch storageBatch) { - var mongoStorageBatch = storageBatch as MongoStorageBatch; - if (globalState.IndexModeCompleted) { if (globalState.IbdMode() == false) { if (globalState.LocalMempoolView.Any()) { - var toRemoveFromMempool = mongoStorageBatch.TransactionBlockTable.Select(s => s.TransactionId).ToList(); + var toRemoveFromMempool = storageBatch.TransactionBlockTable.Select(s => s.TransactionId).ToList(); FilterDefinitionBuilder builder = Builders.Filter; FilterDefinition filter = builder.In(mempoolItem => mempoolItem.TransactionId, @@ -149,24 +146,24 @@ public SyncBlockInfo PushStorageBatch(StorageBatch storageBatch) } } - var blockTableTask = mongoStorageBatch.BlockTable.Values.Any() - ? db.BlockTable.InsertManyAsync(mongoStorageBatch.BlockTable.Values, new InsertManyOptions { IsOrdered = false }) + var blockTableTask = storageBatch.BlockTable.Values.Any() + ? db.BlockTable.InsertManyAsync(storageBatch.BlockTable.Values, new InsertManyOptions { IsOrdered = false }) : Task.CompletedTask; - var transactionBlockTableTask = mongoStorageBatch.TransactionBlockTable.Any() - ? db.TransactionBlockTable.InsertManyAsync(mongoStorageBatch.TransactionBlockTable, new InsertManyOptions { IsOrdered = false }) + var transactionBlockTableTask = storageBatch.TransactionBlockTable.Any() + ? db.TransactionBlockTable.InsertManyAsync(storageBatch.TransactionBlockTable, new InsertManyOptions { IsOrdered = false }) : Task.CompletedTask; - var outputTableTask = mongoStorageBatch.OutputTable.Any() - ? db.OutputTable.InsertManyAsync(mongoStorageBatch.OutputTable.Values, new InsertManyOptions { IsOrdered = false }) + var outputTableTask = storageBatch.OutputTable.Any() + ? db.OutputTable.InsertManyAsync(storageBatch.OutputTable.Values, new InsertManyOptions { IsOrdered = false }) : Task.CompletedTask; Task transactionTableTask = Task.Run(() => { try { - if (mongoStorageBatch.TransactionTable.Any()) - db.TransactionTable.InsertMany(mongoStorageBatch.TransactionTable, new InsertManyOptions {IsOrdered = false}); + if (storageBatch.TransactionTable.Any()) + db.TransactionTable.InsertMany(storageBatch.TransactionTable, new InsertManyOptions {IsOrdered = false}); } catch (MongoBulkWriteException mbwex) { @@ -179,9 +176,9 @@ public SyncBlockInfo PushStorageBatch(StorageBatch storageBatch) } }); - var utxos = new List(mongoStorageBatch.OutputTable.Values.Count); + var utxos = new List(storageBatch.OutputTable.Values.Count); - foreach (OutputTable outputTable in mongoStorageBatch.OutputTable.Values) + foreach (OutputTable outputTable in storageBatch.OutputTable.Values) { if (outputTable.Address.Equals(OpReturnAddress)) continue; @@ -204,14 +201,14 @@ public SyncBlockInfo PushStorageBatch(StorageBatch storageBatch) : Task.CompletedTask; var inputTableTask = Task.CompletedTask; - if (mongoStorageBatch.InputTable.Any()) + if (storageBatch.InputTable.Any()) { var utxosLookups = FetchUtxos( - mongoStorageBatch.InputTable + storageBatch.InputTable .Where(_ => _.Address == null) .Select(_ => _.Outpoint)); - foreach (InputTable input in mongoStorageBatch.InputTable) + foreach (InputTable input in storageBatch.InputTable) { if (input.Address != null) continue; @@ -220,17 +217,17 @@ public SyncBlockInfo PushStorageBatch(StorageBatch storageBatch) input.Value = utxosLookups[key].Value; } - inputTableTask = db.InputTable.InsertManyAsync(mongoStorageBatch.InputTable, new InsertManyOptions { IsOrdered = false }); + inputTableTask = db.InputTable.InsertManyAsync(storageBatch.InputTable, new InsertManyOptions { IsOrdered = false }); } Task.WaitAll(blockTableTask, transactionBlockTableTask, outputTableTask, inputTableTask, transactionTableTask, unspentOutputTableTask); - if (mongoStorageBatch.InputTable.Any()) + if (storageBatch.InputTable.Any()) { // TODO: if earlier we filtered out outputs that are already spent and not pushed to the utxo table // now we do not need to try and delete such outputs becuase they where never pushed to the store. - var outpointsFromNewInput = mongoStorageBatch.InputTable + var outpointsFromNewInput = storageBatch.InputTable .Select(_ => _.Outpoint) .ToList(); @@ -249,7 +246,7 @@ public SyncBlockInfo PushStorageBatch(StorageBatch storageBatch) string lastBlockHash = null; long blockIndex = 0; var markBlocksAsComplete = new List>(); - foreach (BlockTable mapBlock in mongoStorageBatch.BlockTable.Values.OrderBy(b => b.BlockIndex)) + foreach (BlockTable mapBlock in storageBatch.BlockTable.Values.OrderBy(b => b.BlockIndex)) { FilterDefinition filter = Builders.Filter.Eq(block => block.BlockIndex, mapBlock.BlockIndex); @@ -274,7 +271,7 @@ public SyncBlockInfo PushStorageBatch(StorageBatch storageBatch) return block; } - public void InsertMempoolTransactions(SyncBlockTransactionsOperation item) + public InsertStats InsertMempoolTransactions(SyncBlockTransactionsOperation item) { var mempool = new List(); var inputs = new Dictionary(); @@ -352,6 +349,8 @@ public void InsertMempoolTransactions(SyncBlockTransactionsOperation item) foreach (MempoolTable mempooltrx in mempool) globalState.LocalMempoolView.TryAdd(mempooltrx.TransactionId, string.Empty); + + return new InsertStats { Items = mempool }; } protected virtual void OnAddToStorageBatch(StorageBatch storageBatch, SyncBlockTransactionsOperation item) diff --git a/src/Blockcore.Indexer.Core/Storage/Mongo/Types/AddressUtxoComputedTable.cs b/src/Blockcore.Indexer.Core/Storage/Mongo/Types/AddressUtxoComputedTable.cs index e3278b1d..b3a3d5d8 100644 --- a/src/Blockcore.Indexer.Core/Storage/Mongo/Types/AddressUtxoComputedTable.cs +++ b/src/Blockcore.Indexer.Core/Storage/Mongo/Types/AddressUtxoComputedTable.cs @@ -1,5 +1,3 @@ -using Blockcore.Indexer.Core.Storage.Types; - namespace Blockcore.Indexer.Core.Storage.Mongo.Types { /// diff --git a/src/Blockcore.Indexer.Core/Storage/Mongo/Types/InputTable.cs b/src/Blockcore.Indexer.Core/Storage/Mongo/Types/InputTable.cs index a8ccfb04..d65bbc60 100644 --- a/src/Blockcore.Indexer.Core/Storage/Mongo/Types/InputTable.cs +++ b/src/Blockcore.Indexer.Core/Storage/Mongo/Types/InputTable.cs @@ -1,5 +1,3 @@ -using Blockcore.Indexer.Core.Storage.Types; - namespace Blockcore.Indexer.Core.Storage.Mongo.Types { public class InputTable diff --git a/src/Blockcore.Indexer.Core/Storage/Mongo/Types/MempoolTable.cs b/src/Blockcore.Indexer.Core/Storage/Mongo/Types/MempoolTable.cs index 768b7888..72e45f1b 100644 --- a/src/Blockcore.Indexer.Core/Storage/Mongo/Types/MempoolTable.cs +++ b/src/Blockcore.Indexer.Core/Storage/Mongo/Types/MempoolTable.cs @@ -1,5 +1,4 @@ using System.Collections.Generic; -using Blockcore.Indexer.Core.Storage.Types; namespace Blockcore.Indexer.Core.Storage.Mongo.Types { diff --git a/src/Blockcore.Indexer.Core/Storage/Mongo/Types/MongoStorageBatch.cs b/src/Blockcore.Indexer.Core/Storage/Mongo/Types/MongoStorageBatch.cs deleted file mode 100644 index 255f8011..00000000 --- a/src/Blockcore.Indexer.Core/Storage/Mongo/Types/MongoStorageBatch.cs +++ /dev/null @@ -1,43 +0,0 @@ -using System.Collections.Generic; -using System.Linq; -using Blockcore.Indexer.Core.Operations.Types; - -namespace Blockcore.Indexer.Core.Storage.Mongo.Types; - -public class MongoStorageBatch : StorageBatch -{ - public long TotalSize { get; set; } - public List TransactionBlockTable { get; set; } = new(); - public Dictionary BlockTable { get; set; } = new(); - public List TransactionTable { get; set; } = new(); - public Dictionary OutputTable { get; set; } = new(); - public List InputTable { get; set; } = new(); - - public override int GetBlockCount() => BlockTable.Count; - - public override int GetOutputCount() => OutputTable.Count; - - public override int GetInputCount() => InputTable.Count; - - public override int GetTransactionCount() => TransactionBlockTable.Count; - - public override long GetBatchSize() => TotalSize; - - public override IEnumerable GetBlockSizes() => BlockTable.Values.Select(x => x.BlockSize).ToList(); - - public override bool ValidateBatch(string prevBlockHash) - { - string prevHash = prevBlockHash; - foreach (var mapBlock in BlockTable.Values.OrderBy(b => b.BlockIndex)) - { - if (mapBlock.PreviousBlockHash != prevHash) - { - return false; - } - - prevHash = mapBlock.BlockHash; - } - - return true; - } -} diff --git a/src/Blockcore.Indexer.Core/Storage/Types/Outpoint.cs b/src/Blockcore.Indexer.Core/Storage/Mongo/Types/Outpoint.cs similarity index 81% rename from src/Blockcore.Indexer.Core/Storage/Types/Outpoint.cs rename to src/Blockcore.Indexer.Core/Storage/Mongo/Types/Outpoint.cs index 1bf86263..8ae6ff15 100644 --- a/src/Blockcore.Indexer.Core/Storage/Types/Outpoint.cs +++ b/src/Blockcore.Indexer.Core/Storage/Mongo/Types/Outpoint.cs @@ -1,4 +1,4 @@ -namespace Blockcore.Indexer.Core.Storage.Types +namespace Blockcore.Indexer.Core.Storage.Mongo.Types { public class Outpoint { diff --git a/src/Blockcore.Indexer.Core/Storage/Mongo/Types/OutputTable.cs b/src/Blockcore.Indexer.Core/Storage/Mongo/Types/OutputTable.cs index 03829feb..f4fac3fc 100644 --- a/src/Blockcore.Indexer.Core/Storage/Mongo/Types/OutputTable.cs +++ b/src/Blockcore.Indexer.Core/Storage/Mongo/Types/OutputTable.cs @@ -1,9 +1,15 @@ -using Blockcore.Indexer.Core.Storage.Types; - namespace Blockcore.Indexer.Core.Storage.Mongo.Types { - public class OutputTable : Output + public class OutputTable { + public Outpoint Outpoint { get; set; } + + public string Address { get; set; } + public string ScriptHex { get; set; } + public long Value { get; set; } + public uint BlockIndex { get; set; } + public bool CoinBase { get; set; } + public bool CoinStake { get; set; } } } diff --git a/src/Blockcore.Indexer.Core/Storage/Mongo/Types/UnspentOutputTable.cs b/src/Blockcore.Indexer.Core/Storage/Mongo/Types/UnspentOutputTable.cs index b532027c..973c8302 100644 --- a/src/Blockcore.Indexer.Core/Storage/Mongo/Types/UnspentOutputTable.cs +++ b/src/Blockcore.Indexer.Core/Storage/Mongo/Types/UnspentOutputTable.cs @@ -1,5 +1,3 @@ -using Blockcore.Indexer.Core.Storage.Types; - namespace Blockcore.Indexer.Core.Storage.Mongo.Types; public class UnspentOutputTable diff --git a/src/Blockcore.Indexer.Core/Storage/Types/Output.cs b/src/Blockcore.Indexer.Core/Storage/Types/Output.cs deleted file mode 100644 index d313766b..00000000 --- a/src/Blockcore.Indexer.Core/Storage/Types/Output.cs +++ /dev/null @@ -1,14 +0,0 @@ -namespace Blockcore.Indexer.Core.Storage.Types; - -public class Output -{ - public Outpoint Outpoint { get; set; } - - public string Address { get; set; } - - public string ScriptHex { get; set; } - public long Value { get; set; } - public uint BlockIndex { get; set; } - public bool CoinBase { get; set; } - public bool CoinStake { get; set; } -} diff --git a/src/Blockcore.Indexer.Core/Storage/Types/PeerDetails.cs b/src/Blockcore.Indexer.Core/Storage/Types/PeerDetails.cs deleted file mode 100644 index 0e4e79ec..00000000 --- a/src/Blockcore.Indexer.Core/Storage/Types/PeerDetails.cs +++ /dev/null @@ -1,8 +0,0 @@ -using Blockcore.Indexer.Core.Client.Types; - -namespace Blockcore.Indexer.Core.Storage.Types; - -public class PeerDetails : PeerInfo -{ - -} diff --git a/src/Blockcore.Indexer.Core/Sync/SyncOperations.cs b/src/Blockcore.Indexer.Core/Sync/SyncOperations.cs index 98838c9b..19038a39 100644 --- a/src/Blockcore.Indexer.Core/Sync/SyncOperations.cs +++ b/src/Blockcore.Indexer.Core/Sync/SyncOperations.cs @@ -1,6 +1,7 @@ using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; +using Blockcore.Consensus.BlockInfo; using Blockcore.Consensus.ScriptInfo; using Blockcore.Consensus.TransactionInfo; using Blockcore.Indexer.Core.Client; @@ -9,10 +10,13 @@ using Blockcore.Indexer.Core.Operations.Types; using Blockcore.Indexer.Core.Settings; using Blockcore.Indexer.Core.Storage; +using Blockcore.Indexer.Core.Storage.Mongo; +using Blockcore.Indexer.Core.Storage.Mongo.Types; using Blockcore.Indexer.Core.Storage.Types; using Microsoft.Extensions.Caching.Memory; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; +using MongoDB.Driver; namespace Blockcore.Indexer.Core.Sync { @@ -22,6 +26,7 @@ namespace Blockcore.Indexer.Core.Sync public class SyncOperations : ISyncOperations { private readonly IStorage storage; + private readonly IMongoDb db; private readonly ILogger log; @@ -45,7 +50,7 @@ public SyncOperations( IOptions configuration, IMemoryCache cache, GlobalState globalState, ICryptoClientFactory clientFactory, - ISyncBlockTransactionOperationBuilder blockInfoEnrichment) + ISyncBlockTransactionOperationBuilder blockInfoEnrichment, IMongoDb db) { this.configuration = configuration.Value; log = logger; @@ -54,6 +59,7 @@ public SyncOperations( this.globalState = globalState; this.clientFactory = clientFactory; transactionOperationBuilder = blockInfoEnrichment; + this.db = db; // Register the cold staking template. StandardScripts.RegisterStandardScriptTemplate(ColdStakingScriptTemplate.Instance); @@ -63,11 +69,11 @@ public SyncOperations( public void InitializeMmpool() { - var mempoolTransactionIds = storage.GetMempoolTransactionIds(); + var allitems = db.Mempool.AsQueryable().ToList(); - foreach (var transactionId in mempoolTransactionIds) + foreach (MempoolTable allitem in allitems) { - globalState.LocalMempoolView.TryAdd(transactionId, string.Empty); + globalState.LocalMempoolView.TryAdd(allitem.TransactionId, string.Empty); } } @@ -163,7 +169,10 @@ private SyncPoolTransactions FindPoolInternal(SyncConnection connection) { List toRemoveFromMempool = deleteTransaction; - storage.DeleteTransactionsFromMempool(toRemoveFromMempool); + FilterDefinitionBuilder builder = Builders.Filter; + FilterDefinition filter = builder.In(mempoolItem => mempoolItem.TransactionId, toRemoveFromMempool); + + db.Mempool.DeleteMany(filter); foreach (string mempooltrx in toRemoveFromMempool) globalState.LocalMempoolView.Remove(mempooltrx, out _); @@ -227,7 +236,7 @@ private SyncBlockTransactionsOperation SyncBlockInternal(SyncConnection connecti string hex = client.GetBlockHex(block.Hash); - var blockItem = Consensus.BlockInfo.Block.Parse(hex, connection.Network.Consensus.ConsensusFactory); + var blockItem = Block.Parse(hex, connection.Network.Consensus.ConsensusFactory); foreach (Transaction blockItemTransaction in blockItem.Transactions) { diff --git a/src/Blockcore.Indexer.Core/Storage/Mongo/SyncTasks/MongoDbBlockIndexer.cs b/src/Blockcore.Indexer.Core/Sync/SyncTasks/BlockIndexer.cs similarity index 96% rename from src/Blockcore.Indexer.Core/Storage/Mongo/SyncTasks/MongoDbBlockIndexer.cs rename to src/Blockcore.Indexer.Core/Sync/SyncTasks/BlockIndexer.cs index c393ca3f..80051f76 100644 --- a/src/Blockcore.Indexer.Core/Storage/Mongo/SyncTasks/MongoDbBlockIndexer.cs +++ b/src/Blockcore.Indexer.Core/Sync/SyncTasks/BlockIndexer.cs @@ -4,19 +4,23 @@ using System.Text; using System.Threading.Tasks; using Blockcore.Indexer.Core.Extensions; +using Blockcore.Indexer.Core.Operations; +using Blockcore.Indexer.Core.Operations.Types; using Blockcore.Indexer.Core.Settings; +using Blockcore.Indexer.Core.Storage; +using Blockcore.Indexer.Core.Storage.Mongo; using Blockcore.Indexer.Core.Storage.Mongo.Types; -using Blockcore.Indexer.Core.Sync.SyncTasks; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; +using MongoDB.Bson; using MongoDB.Driver; -namespace Blockcore.Indexer.Core.Storage.Mongo.SyncTasks +namespace Blockcore.Indexer.Core.Sync.SyncTasks { - public class MongoDbBlockIndexer : TaskRunner + public class BlockIndexer : TaskRunner { private readonly IndexerSettings config; - private readonly ILogger log; + private readonly ILogger log; readonly IStorage data; private readonly System.Diagnostics.Stopwatch watch; @@ -27,9 +31,9 @@ public class MongoDbBlockIndexer : TaskRunner Task indexingCompletTask; bool initialized; - public MongoDbBlockIndexer( + public BlockIndexer( IOptions configuration, - ILogger logger, + ILogger logger, IStorage data, IMongoDb db) : base(configuration, logger) { diff --git a/src/Blockcore.Indexer.Core/Sync/SyncTasks/BlockPuller.cs b/src/Blockcore.Indexer.Core/Sync/SyncTasks/BlockPuller.cs index fe0b32e2..ccdbf2f5 100644 --- a/src/Blockcore.Indexer.Core/Sync/SyncTasks/BlockPuller.cs +++ b/src/Blockcore.Indexer.Core/Sync/SyncTasks/BlockPuller.cs @@ -9,7 +9,6 @@ using Blockcore.Indexer.Core.Operations; using Blockcore.Indexer.Core.Operations.Types; using Blockcore.Indexer.Core.Settings; -using Blockcore.Indexer.Core.Storage; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; @@ -36,8 +35,6 @@ public class BlockPuller : TaskRunner readonly ICryptoClientFactory clientFactory; - readonly IStorageBatchFactory StorageBatchFactory; - private readonly IEnumerable bip30Blocks = new List {91842 , 91880 }; private readonly BlockingCollection pendingBlocksToAddToStorage = new(); @@ -55,14 +52,12 @@ public BlockPuller( SyncConnection syncConnection, ILogger logger, IStorageOperations storageOperations, - ICryptoClientFactory clientFactory, - IStorageBatchFactory storageBatchFactory) + ICryptoClientFactory clientFactory) : base(configuration, logger) { log = logger; this.storageOperations = storageOperations; this.clientFactory = clientFactory; - StorageBatchFactory = storageBatchFactory; this.syncConnection = syncConnection; this.syncOperations = syncOperations; config = configuration.Value; @@ -113,7 +108,7 @@ public override async Task OnExecute() { // start pulling blocks form this tip Runner.GlobalState.PullingTip = await clientFactory.Create(syncConnection).GetBlockAsync(Runner.GlobalState.StoreTip.BlockHash); - currentStorageBatch = StorageBatchFactory.GetStorageBatch(); + currentStorageBatch = new StorageBatch(); log.LogInformation($"Fetching block started at block {Runner.GlobalState.PullingTip.Height}({Runner.GlobalState.PullingTip.Hash})"); } @@ -216,21 +211,21 @@ private void ProcessFromBlockingCollection() bool bip30Issue = bip30Blocks.Contains(block.BlockInfo.Height + 1); if (ibd && !bip30Issue && - currentStorageBatch.GetBlockCount() < config.DbBatchCount && - currentStorageBatch.GetBatchSize() <= config.DbBatchSize) + currentStorageBatch.BlockTable.Count < config.MongoBatchCount && + currentStorageBatch.TotalSize <= config.MongoBatchSize) { continue; } - long totalBlocks = currentStorageBatch.GetBlockCount(); + long totalBlocks = currentStorageBatch.BlockTable.Count; double totalSeconds = watchBatch.Elapsed.TotalSeconds; double blocksPerSecond = totalBlocks / totalSeconds; double secondsPerBlock = totalSeconds / totalBlocks; - log.LogInformation($"Puller - blocks={currentStorageBatch.GetBlockCount()}, height = {block.BlockInfo.Height}, batch size = {((decimal)currentStorageBatch.GetBatchSize() / 1000000):0.00}mb, Seconds = {watchBatch.Elapsed.TotalSeconds}, fetchs = {blocksPerSecond:0.00}b/s ({secondsPerBlock:0.00}s/b). ({pendingBlocksToAddToStorage.Count})"); + log.LogInformation($"Puller - blocks={currentStorageBatch.BlockTable.Count}, height = {block.BlockInfo.Height}, batch size = {((decimal)currentStorageBatch.TotalSize / 1000000):0.00}mb, Seconds = {watchBatch.Elapsed.TotalSeconds}, fetchs = {blocksPerSecond:0.00}b/s ({secondsPerBlock:0.00}s/b). ({pendingBlocksToAddToStorage.Count})"); Runner.Get().Enqueue(currentStorageBatch); - currentStorageBatch = StorageBatchFactory.GetStorageBatch(); + currentStorageBatch = new StorageBatch(); watchBatch.Restart(); } diff --git a/src/Blockcore.Indexer.Core/Sync/SyncTasks/BlockStartup.cs b/src/Blockcore.Indexer.Core/Sync/SyncTasks/BlockStartup.cs index f2b818cc..44befc85 100644 --- a/src/Blockcore.Indexer.Core/Sync/SyncTasks/BlockStartup.cs +++ b/src/Blockcore.Indexer.Core/Sync/SyncTasks/BlockStartup.cs @@ -7,6 +7,7 @@ using Blockcore.Indexer.Core.Operations.Types; using Blockcore.Indexer.Core.Settings; using Blockcore.Indexer.Core.Storage; +using Blockcore.Indexer.Core.Storage.Mongo; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; @@ -24,9 +25,8 @@ public class BlockStartup : TaskStarter private readonly SyncConnection connection; private readonly IStorageOperations storageOperations; readonly ICryptoClientFactory clientFactory; - private readonly IStorage storageData; + private readonly MongoData mongoData; readonly IOptions indexerSettings; - readonly IStorageBatchFactory StorageBatchFactory; /// /// Initializes a new instance of the class. @@ -38,19 +38,17 @@ public BlockStartup( IStorageOperations storageOperations, ICryptoClientFactory clientFactory, IStorage data, - IOptions indexerSettings, - IStorageBatchFactory storageBatchFactory) + IOptions indexerSettings) : base(logger) { connection = syncConnection; this.storageOperations = storageOperations; this.clientFactory = clientFactory; this.indexerSettings = indexerSettings; - StorageBatchFactory = storageBatchFactory; this.syncOperations = syncOperations; log = logger; - storageData = data; + mongoData = (MongoData)data; } @@ -69,7 +67,7 @@ public override async Task OnExecute() { IBlockchainClient client = clientFactory.Create(connection); - List allIndexes = storageData.GetBlockIndexIndexes(); + List allIndexes = mongoData.GetBlockIndexIndexes(); if (allIndexes.Count == indexerSettings.Value.IndexCountForBlockIndexProperty) { @@ -94,7 +92,7 @@ public override async Task OnExecute() BlockInfo genesisBlock = await client.GetBlockAsync(genesisHash); SyncBlockTransactionsOperation block = syncOperations.FetchFullBlock(connection, genesisBlock); - StorageBatch genesisBatch = StorageBatchFactory.GetStorageBatch(); + StorageBatch genesisBatch = new StorageBatch(); storageOperations.AddToStorageBatch(genesisBatch, block); Runner.GlobalState.StoreTip = storageOperations.PushStorageBatch(genesisBatch); } diff --git a/src/Blockcore.Indexer.Core/Sync/SyncTasks/BlockStore.cs b/src/Blockcore.Indexer.Core/Sync/SyncTasks/BlockStore.cs index 36f38d02..3446dc2c 100644 --- a/src/Blockcore.Indexer.Core/Sync/SyncTasks/BlockStore.cs +++ b/src/Blockcore.Indexer.Core/Sync/SyncTasks/BlockStore.cs @@ -6,6 +6,7 @@ using Blockcore.Indexer.Core.Operations; using Blockcore.Indexer.Core.Operations.Types; using Blockcore.Indexer.Core.Settings; +using Blockcore.Indexer.Core.Storage.Mongo.Types; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; @@ -22,6 +23,7 @@ public class BlockStore : TaskRunner private readonly ISyncOperations syncOperations; private readonly SyncConnection syncConnection; + readonly IUtxoCache utxoCache; private readonly System.Diagnostics.Stopwatch watch; @@ -33,12 +35,14 @@ public BlockStore( ILogger logger, IStorageOperations storageOperations, ISyncOperations syncOperations, - SyncConnection syncConnection) + SyncConnection syncConnection, + IUtxoCache utxoCache) : base(configuration, logger) { this.storageOperations = storageOperations; this.syncOperations = syncOperations; this.syncConnection = syncConnection; + this.utxoCache = utxoCache; log = logger; watch = Stopwatch.Start(); } @@ -46,7 +50,7 @@ public BlockStore( /// public override async Task OnExecute() { - if (Runner.GlobalState.ReorgMode) + if (Runner.GlobalState.ReorgMode == true) { // null the store tip so the document count will be taken form disk Runner.GlobalState.StoreTip = null; @@ -67,8 +71,7 @@ public override async Task OnExecute() if (!TryDequeue(out StorageBatch batch)) return await Task.FromResult(false); - if (batch.ValidateBatch(Runner.GlobalState.StoreTip.BlockHash) == false) - throw new ApplicationException("None consecutive block received"); + ValidateBatch(batch); watch.Restart(); @@ -79,15 +82,15 @@ public override async Task OnExecute() if (Runner.GlobalState.StoreTip == null) throw new ApplicationException("Store tip was not persisted"); - long totalBlocks = batch.GetBlockCount();// insertStats.Sum((tuple => tuple.count)); + long totalBlocks = batch.BlockTable.Count;// insertStats.Sum((tuple => tuple.count)); double totalSeconds = watch.Elapsed.TotalSeconds;// insertStats.Sum((tuple => tuple.seconds)); double blocksPerSecond = totalBlocks / totalSeconds; double secondsPerBlock = totalSeconds / totalBlocks; - log.LogInformation($"Store - blocks={batch.GetBlockCount()}, outputs={batch.GetOutputCount()}, inputs={batch.GetInputCount()}, trx={batch.GetTransactionCount()}, total Size = {((decimal)batch.GetBatchSize() / 1000000):0.00}mb, tip={Runner.GlobalState.StoreTip.BlockIndex}, Seconds = {watch.Elapsed.TotalSeconds}, inserts = {blocksPerSecond:0.00}b/s ({secondsPerBlock:0.00}s/b)"); + log.LogInformation($"Store - blocks={batch.BlockTable.Count}, outputs={batch.OutputTable.Count}, inputs={batch.InputTable.Count}, trx={batch.TransactionBlockTable.Count}, total Size = {((decimal)batch.TotalSize / 1000000):0.00}mb, tip={Runner.GlobalState.StoreTip.BlockIndex}, Seconds = {watch.Elapsed.TotalSeconds}, inserts = {blocksPerSecond:0.00}b/s ({secondsPerBlock:0.00}s/b)"); - foreach (var blockSiz in batch.GetBlockSizes()) - syncConnection.RecentItems.Add((DateTime.UtcNow, TimeSpan.FromSeconds(blocksPerSecond), blockSiz)); + foreach (BlockTable mapBlocksValue in batch.BlockTable.Values) + syncConnection.RecentItems.Add((DateTime.UtcNow, TimeSpan.FromSeconds(blocksPerSecond), mapBlocksValue.BlockSize)); var notifications = new AddressNotifications { Addresses = new List() };// count.Items.Where(ad => ad.Addresses != null).SelectMany(s => s.Addresses).Distinct().ToList() }; Runner.Get().Enqueue(notifications); @@ -95,5 +98,20 @@ public override async Task OnExecute() return await Task.FromResult(true); } + + void ValidateBatch(StorageBatch item) + { + // check all blocks are consecutive and start from the last block in store. + string prevHash = Runner.GlobalState.StoreTip.BlockHash; + foreach (BlockTable mapBlock in item.BlockTable.Values.OrderBy(b => b.BlockIndex)) + { + if (mapBlock.PreviousBlockHash != prevHash) + { + throw new ApplicationException("None consecutive block received"); + } + + prevHash = mapBlock.BlockHash; + } + } } } diff --git a/src/Blockcore.Indexer.Core/Storage/Mongo/SyncTasks/RichListScanning.cs b/src/Blockcore.Indexer.Core/Sync/SyncTasks/RichListSync.cs similarity index 95% rename from src/Blockcore.Indexer.Core/Storage/Mongo/SyncTasks/RichListScanning.cs rename to src/Blockcore.Indexer.Core/Sync/SyncTasks/RichListSync.cs index b0730f06..cf800103 100644 --- a/src/Blockcore.Indexer.Core/Storage/Mongo/SyncTasks/RichListScanning.cs +++ b/src/Blockcore.Indexer.Core/Sync/SyncTasks/RichListSync.cs @@ -4,19 +4,21 @@ using System.Linq; using System.Threading.Tasks; using Blockcore.Indexer.Core.Settings; +using Blockcore.Indexer.Core.Storage; +using Blockcore.Indexer.Core.Storage.Mongo; using Blockcore.Indexer.Core.Storage.Mongo.Types; -using Blockcore.Indexer.Core.Sync.SyncTasks; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using MongoDB.Bson; using MongoDB.Driver; +using MongoDB.Driver.Core.Operations; -namespace Blockcore.Indexer.Core.Storage.Mongo.SyncTasks +namespace Blockcore.Indexer.Core.Sync.SyncTasks { - public class RichListScanning : TaskRunner + public class RichListSync : TaskRunner { private readonly IMongoDb db; - private readonly ILogger log; + private readonly ILogger log; private readonly IndexerSettings indexerSettings; private readonly Stopwatch watch; @@ -24,7 +26,7 @@ public class RichListScanning : TaskRunner private bool syncInProgress; DateTime lastSync; - public RichListScanning(IOptions configuration, ILogger logger,IMongoDb data) + public RichListSync(IOptions configuration, ILogger logger,IMongoDb data) : base(configuration, logger) { db = data; diff --git a/src/Blockcore.Indexer.Core/Sync/SyncTasks/StatsSyncer.cs b/src/Blockcore.Indexer.Core/Sync/SyncTasks/StatsSyncer.cs index 1aa00581..f3b89689 100644 --- a/src/Blockcore.Indexer.Core/Sync/SyncTasks/StatsSyncer.cs +++ b/src/Blockcore.Indexer.Core/Sync/SyncTasks/StatsSyncer.cs @@ -8,7 +8,7 @@ using Blockcore.Indexer.Core.Operations.Types; using Blockcore.Indexer.Core.Settings; using Blockcore.Indexer.Core.Storage; -using Blockcore.Indexer.Core.Storage.Types; +using Blockcore.Indexer.Core.Storage.Mongo; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; @@ -25,7 +25,7 @@ public class StatsSyncer : TaskRunner private readonly IStorageOperations storageOperations; - private readonly IStorage data; + private readonly MongoData data; private readonly System.Diagnostics.Stopwatch watch; @@ -43,7 +43,7 @@ public StatsSyncer( { log = logger; - data = storage; + data = (MongoData)storage; this.statsHandler = statsHandler; this.storageOperations = storageOperations; @@ -67,28 +67,7 @@ public override async Task OnExecute() // TODO: Look into potential optimization on updates, not replacing the whole document for all connected nodes all the time. foreach (PeerInfo peer in peers) { - await data.InsertPeer(new PeerDetails - { - Addr = peer.Addr, - AddrLocal = peer.AddrLocal, - Version = peer.Version, - Inbound = peer.Inbound, - Services = peer.Services, - BanScore = peer.BanScore, - BytesRecv = peer.BytesRecv, - BytesSent = peer.BytesSent, - ConnTime = peer.ConnTime, - PingTime = peer.PingTime, - InFlight = peer.InFlight, - LastRecv = peer.LastRecv, - LastSeen = peer.LastSeen, - LastSend = peer.LastSend, - StartingHeight = peer.StartingHeight, - SubVer = peer.SubVer, - SyncedBlocks = peer.SyncedBlocks, - SyncedHeaders = peer.SyncedHeaders, - WhiteListed = peer.WhiteListed - }); + await data.InsertPeer(peer); } log.LogInformation($"Time taken to update peers in database: {watch.Elapsed.TotalSeconds}."); diff --git a/src/Blockcore.Indexer.Tests/Storage/Mongo/MongoDataTests.cs b/src/Blockcore.Indexer.Tests/Storage/Mongo/MongoDataTests.cs index cf9fdb75..81ed5aef 100644 --- a/src/Blockcore.Indexer.Tests/Storage/Mongo/MongoDataTests.cs +++ b/src/Blockcore.Indexer.Tests/Storage/Mongo/MongoDataTests.cs @@ -11,7 +11,6 @@ using Blockcore.Indexer.Core.Storage; using Blockcore.Indexer.Core.Storage.Mongo; using Blockcore.Indexer.Core.Storage.Mongo.Types; -using Blockcore.Indexer.Core.Storage.Types; using Blockcore.Networks; using Microsoft.Extensions.Options; using Moq; diff --git a/src/Blockcore.Indexer.Tests/Storage/Mongo/MongoStorageOperationsTests.cs b/src/Blockcore.Indexer.Tests/Storage/Mongo/MongoStorageOperationsTests.cs index 70522640..99b7c37c 100644 --- a/src/Blockcore.Indexer.Tests/Storage/Mongo/MongoStorageOperationsTests.cs +++ b/src/Blockcore.Indexer.Tests/Storage/Mongo/MongoStorageOperationsTests.cs @@ -14,7 +14,6 @@ using Blockcore.Indexer.Core.Storage; using Blockcore.Indexer.Core.Storage.Mongo; using Blockcore.Indexer.Core.Storage.Mongo.Types; -using Blockcore.Indexer.Core.Storage.Types; using Blockcore.NBitcoin; using Blockcore.Networks; using FluentAssertions; @@ -87,6 +86,7 @@ public MongoStorageOperationsTests() sut = new MongoStorageOperations(syncConnection, mongodbMock.MongoDbObject, + new UtxoCache(null), indexSettingsMock.Object, globalState, new MapMongoBlockToStorageBlock(), @@ -154,9 +154,9 @@ static SyncBlockTransactionsOperation WithRandomSyncBlockTransactionsOperation() return item; } - MongoStorageBatch WithBatchThatHasABlockToPushAndUpdatesToSyncCompleteSuccessfully() + StorageBatch WithBatchThatHasABlockToPushAndUpdatesToSyncCompleteSuccessfully() { - var batch = new MongoStorageBatch(); + var batch = new StorageBatch(); var block = NewRandomBlockTable; batch.BlockTable.Add(block.BlockIndex, block); @@ -167,7 +167,7 @@ MongoStorageBatch WithBatchThatHasABlockToPushAndUpdatesToSyncCompleteSuccessful return batch; } - void WithASuccessfulDeleteManyOnUnspentOutputTable(MongoStorageBatch batch) + void WithASuccessfulDeleteManyOnUnspentOutputTable(StorageBatch batch) { var utxo = batch.InputTable.Select(_ => _.Outpoint).ToList(); @@ -181,7 +181,7 @@ void WithASuccessfulDeleteManyOnUnspentOutputTable(MongoStorageBatch batch) [Fact] public void AddToStorageBatchSetsTheTotalSizeFromBlockInfo() { - var batch = new MongoStorageBatch(); + var batch = new StorageBatch(); var item = WithRandomSyncBlockTransactionsOperation(); @@ -193,7 +193,7 @@ public void AddToStorageBatchSetsTheTotalSizeFromBlockInfo() [Fact] public void AddToStorageBatchSetsTheBlockTableFromBlockInfo() { - var batch = new MongoStorageBatch(); + var batch = new StorageBatch(); var item = WithRandomSyncBlockTransactionsOperation(); @@ -215,7 +215,7 @@ public void AddToStorageBatchSetsTheBlockTableFromBlockInfo() [Fact] public void AddToStorageBatchSetsTheTransactionBlockTableFromTransactions() { - var batch = new MongoStorageBatch(); + var batch = new StorageBatch(); var item = WithRandomSyncBlockTransactionsOperation(); @@ -232,7 +232,7 @@ public void AddToStorageBatchSetsTheTransactionBlockTableFromTransactions() [Fact] public void AddToStorageBatchWithStoreRawTransactionsTrueSetsTransactionTable() { - var batch = new MongoStorageBatch(); + var batch = new StorageBatch(); var item = WithRandomSyncBlockTransactionsOperation(); @@ -253,7 +253,7 @@ public void AddToStorageBatchWithStoreRawTransactionsTrueSetsTransactionTable() [Fact] public void AddToStorageBatchSetsTheOutputsInTheTransactionToTheOutputTable() { - var batch = new MongoStorageBatch(); + var batch = new StorageBatch(); var valueMoney = new Money(NewRandomInt32); var script = new Script(NewRandomString.Replace('-','1')); var item = WithRandomSyncBlockTransactionsOperation(); @@ -299,7 +299,7 @@ public void AddToStorageBatchSetsTheOutputsInTheTransactionToTheOutputTable() [Fact] public void AddToStorageBatchSetsTheInputsInInputTableWithoutAddress() { - var batch = new MongoStorageBatch(); + var batch = new StorageBatch(); var hash = new uint256($"{NewRandomInt64}{NewRandomInt64}{NewRandomInt64}{NewRandomInt64}".Substring(0,64)); int n = NewRandomInt32; var expectedOutpoint = new OutPoint @@ -339,7 +339,7 @@ public void AddToStorageBatchSetsTheInputsInInputTableWithoutAddress() [Fact] public void AddToStorageBatchSetsTheInputsInInputTableWithAddressAndValue() { - var batch = new MongoStorageBatch(); + var batch = new StorageBatch(); var valueMoney = new Money(NewRandomInt32); var script = new Script(NewRandomString.Replace('-','1')); var transaction = new Transaction { Outputs = { { new TxOut { Value = valueMoney, ScriptPubKey = script } } } }; @@ -370,7 +370,7 @@ public void AddToStorageBatchSetsTheInputsInInputTableWithAddressAndValue() [Fact] public void PushStorageBatchAddsBlockTableFromBatchToMongoDb() { - var batch = new MongoStorageBatch(); + var batch = new StorageBatch(); var block = NewRandomBlockTable; batch.BlockTable.Add(block.BlockIndex, block); @@ -387,7 +387,7 @@ public void PushStorageBatchAddsBlockTableFromBatchToMongoDb() [Fact] public void PushStorageBatchAddsTransactionBlockTableFromBatchToMongoDb() { - MongoStorageBatch batch = WithBatchThatHasABlockToPushAndUpdatesToSyncCompleteSuccessfully(); + StorageBatch batch = WithBatchThatHasABlockToPushAndUpdatesToSyncCompleteSuccessfully(); var blockTable = new TransactionBlockTable { @@ -406,7 +406,7 @@ public void PushStorageBatchAddsTransactionBlockTableFromBatchToMongoDb() [Fact] public void PushStorageBatchAddsOutputTableFromBatchToMongoDb() { - MongoStorageBatch batch = WithBatchThatHasABlockToPushAndUpdatesToSyncCompleteSuccessfully(); + StorageBatch batch = WithBatchThatHasABlockToPushAndUpdatesToSyncCompleteSuccessfully(); var outputTable = new OutputTable() { @@ -430,7 +430,7 @@ public void PushStorageBatchAddsOutputTableFromBatchToMongoDb() [Fact] public void PushStorageBatchAddsInputTableAddsToMongodb() { - MongoStorageBatch batch = WithBatchThatHasABlockToPushAndUpdatesToSyncCompleteSuccessfully(); + StorageBatch batch = WithBatchThatHasABlockToPushAndUpdatesToSyncCompleteSuccessfully(); //We must have an unspent output for an input that is being processed var outpoint = new Outpoint { OutputIndex = NewRandomInt32, TransactionId = NewRandomString }; @@ -466,7 +466,7 @@ public void PushStorageBatchAddsInputTableAddsToMongodb() [Fact] public void PushStorageBatchAddsTransactionTableDataToMongodb() { - MongoStorageBatch batch = WithBatchThatHasABlockToPushAndUpdatesToSyncCompleteSuccessfully(); + StorageBatch batch = WithBatchThatHasABlockToPushAndUpdatesToSyncCompleteSuccessfully(); var transactionTable = new TransactionTable() { @@ -486,7 +486,7 @@ public void PushStorageBatchAddsTransactionTableDataToMongodb() public void PushStorageBatchIgnoresDuplicatsOnInputTableForDuplicateKeyException() #pragma warning restore xUnit1013 // Public method should be marked as test { - MongoStorageBatch batch = WithBatchThatHasABlockToPushAndUpdatesToSyncCompleteSuccessfully(); + StorageBatch batch = WithBatchThatHasABlockToPushAndUpdatesToSyncCompleteSuccessfully(); var transactionTable = new TransactionTable() { @@ -513,7 +513,7 @@ public void PushStorageBatchIgnoresDuplicatsOnInputTableForDuplicateKeyException [Fact] public void PushStorageBatchAddsUnspendOutputToMongodbForEachOutputTableItem() { - MongoStorageBatch batch = WithBatchThatHasABlockToPushAndUpdatesToSyncCompleteSuccessfully(); + StorageBatch batch = WithBatchThatHasABlockToPushAndUpdatesToSyncCompleteSuccessfully(); var outputTable = new OutputTable { @@ -541,7 +541,7 @@ public void PushStorageBatchAddsUnspendOutputToMongodbForEachOutputTableItem() [Fact] public void PushStorageBatchAddsDeletsFromUnspentOutputTableOnMongoDbForAllInputTableItems() { - MongoStorageBatch batch = WithBatchThatHasABlockToPushAndUpdatesToSyncCompleteSuccessfully(); + StorageBatch batch = WithBatchThatHasABlockToPushAndUpdatesToSyncCompleteSuccessfully(); //We must have an unspent output for an input that is being processed var outpoint = new Outpoint { OutputIndex = NewRandomInt32, TransactionId = NewRandomString }; @@ -578,7 +578,7 @@ public void PushStorageBatchAddsDeletsFromUnspentOutputTableOnMongoDbForAllInput [Fact] public void PushStorageBatchUpdatesBlockTableItemsWithSyncCompleteTrueInMongodb() { - MongoStorageBatch batch = WithBatchThatHasABlockToPushAndUpdatesToSyncCompleteSuccessfully(); + StorageBatch batch = WithBatchThatHasABlockToPushAndUpdatesToSyncCompleteSuccessfully(); sut.PushStorageBatch(batch); @@ -602,7 +602,7 @@ public void PushStorageBatchUpdatesBlockTableItemsWithSyncCompleteTrueInMongodb( [Fact] public void PushStorageBatchThrowsWhenTheBlockInertedIsNotTheSameHashAsTopBlockInMongodb() { - var batch = new MongoStorageBatch(); + var batch = new StorageBatch(); var dbBlock = NewRandomBlockTable; var storageBlock = NewRandomBlockTable; diff --git a/src/Blockcore.Indexer/appsettings.json b/src/Blockcore.Indexer/appsettings.json index b646ac7a..c67bc07e 100644 --- a/src/Blockcore.Indexer/appsettings.json +++ b/src/Blockcore.Indexer/appsettings.json @@ -18,7 +18,6 @@ } }, "Indexer": { - "DbType": "MongoDb", "ConnectionString": "mongodb://{Symbol}-mongo", "DatabaseNameSubfix": true, @@ -34,8 +33,8 @@ // Syncing parameters "SyncBlockchain": true, "SyncMemoryPool": true, - "DbBatchSize": 10000000, - "DbBatchCount" : 10000, + "MongoBatchSize": 10000000, + "MongoBatchCount" : 10000, "ParallelRequestsToTransactionRpc": 50, "DetailedTrace": 0, "MaxItemsInQueue": 10,