LMDB [DO NOT MERGE]

Builds/CMake/RippledCore.cmake

@@ -417,6 +417,7 @@ target_sources (rippled PRIVATE
   src/ripple/app/paths/impl/XRPEndpointStep.cpp
   src/ripple/app/rdb/backend/detail/impl/Node.cpp
   src/ripple/app/rdb/backend/detail/impl/Shard.cpp
+  src/ripple/app/rdb/backend/impl/LMDBDatabase.cpp
   src/ripple/app/rdb/backend/impl/PostgresDatabase.cpp
   src/ripple/app/rdb/backend/impl/SQLiteDatabase.cpp
   src/ripple/app/rdb/impl/Download.cpp
@@ -433,13 +434,17 @@ target_sources (rippled PRIVATE
   src/ripple/app/tx/impl/CancelOffer.cpp
   src/ripple/app/tx/impl/CashCheck.cpp
   src/ripple/app/tx/impl/Change.cpp
+  src/ripple/app/tx/impl/ClaimReward.cpp
   src/ripple/app/tx/impl/CreateCheck.cpp
   src/ripple/app/tx/impl/CreateOffer.cpp
   src/ripple/app/tx/impl/CreateTicket.cpp
   src/ripple/app/tx/impl/DeleteAccount.cpp
   src/ripple/app/tx/impl/DepositPreauth.cpp
   src/ripple/app/tx/impl/Escrow.cpp
+  src/ripple/app/tx/impl/GenesisMint.cpp
+  src/ripple/app/tx/impl/Import.cpp
   src/ripple/app/tx/impl/InvariantCheck.cpp
+  src/ripple/app/tx/impl/Invoke.cpp
   src/ripple/app/tx/impl/NFTokenAcceptOffer.cpp
   src/ripple/app/tx/impl/NFTokenBurn.cpp
   src/ripple/app/tx/impl/NFTokenCancelOffer.cpp
@@ -448,14 +453,10 @@ target_sources (rippled PRIVATE
   src/ripple/app/tx/impl/OfferStream.cpp
   src/ripple/app/tx/impl/PayChan.cpp
   src/ripple/app/tx/impl/Payment.cpp
+  src/ripple/app/tx/impl/Remit.cpp
   src/ripple/app/tx/impl/SetAccount.cpp
-  src/ripple/app/tx/impl/SetRegularKey.cpp
   src/ripple/app/tx/impl/SetHook.cpp
-  src/ripple/app/tx/impl/ClaimReward.cpp
-  src/ripple/app/tx/impl/GenesisMint.cpp
-  src/ripple/app/tx/impl/Import.cpp
-  src/ripple/app/tx/impl/Invoke.cpp
-  src/ripple/app/tx/impl/Remit.cpp
+  src/ripple/app/tx/impl/SetRegularKey.cpp
   src/ripple/app/tx/impl/SetSignerList.cpp
   src/ripple/app/tx/impl/SetTrust.cpp
   src/ripple/app/tx/impl/SignerEntries.cpp
@@ -927,6 +928,7 @@ if (tests)
     src/test/nodestore/Backend_test.cpp
     src/test/nodestore/Basics_test.cpp
     src/test/nodestore/DatabaseShard_test.cpp
+    src/test/nodestore/LMDB_test.cpp
     src/test/nodestore/Database_test.cpp
     src/test/nodestore/Timing_test.cpp
     src/test/nodestore/import_test.cpp

Builds/CMake/deps/LMDB.cmake

@@ -0,0 +1,32 @@
+include(FetchContent)
+
+FetchContent_Declare(
+  lmdb
+  GIT_REPOSITORY https://github.com/LMDB/lmdb.git
+  GIT_TAG        LMDB_0.9.31
+)
+
+FetchContent_MakeAvailable(lmdb)
+
+add_library(lmdb STATIC
+  ${lmdb_SOURCE_DIR}/libraries/liblmdb/mdb.c
+  ${lmdb_SOURCE_DIR}/libraries/liblmdb/midl.c
+)
+
+target_include_directories(lmdb PUBLIC ${lmdb_SOURCE_DIR}/libraries/liblmdb)
+
+install(TARGETS lmdb
+  EXPORT lmdbTargets
+  ARCHIVE DESTINATION lib
+  LIBRARY DESTINATION lib
+  RUNTIME DESTINATION bin
+  INCLUDES DESTINATION include
+)
+
+install(DIRECTORY ${lmdb_SOURCE_DIR}/libraries/liblmdb/
+  DESTINATION include
+  FILES_MATCHING PATTERN "*.h"
+)
+
+add_library (NIH::lmdb ALIAS lmdb)
+target_link_libraries (ripple_libs INTERFACE NIH::lmdb)

Builds/levelization/results/loops.txt

@@ -53,7 +53,7 @@ Loop: test.app test.jtx
   test.app > test.jtx
 
 Loop: test.app test.rpc
-  test.rpc == test.app
+  test.rpc ~= test.app
 
 Loop: test.jtx test.toplevel
   test.toplevel > test.jtx

CMakeLists.txt

@@ -75,6 +75,7 @@ include(deps/gRPC)
 include(deps/cassandra)
 include(deps/Postgres)
 include(deps/WasmEdge)
+include(deps/LMDB)
 
 ###
 

src/ripple/app/hook/impl/applyHook.cpp

@@ -1,12 +1,17 @@
 #include <ripple/app/hook/applyHook.h>
 #include <ripple/app/ledger/OpenLedger.h>
 #include <ripple/app/ledger/TransactionMaster.h>
+#include <ripple/app/misc/HashRouter.h>
 #include <ripple/app/misc/NetworkOPs.h>
 #include <ripple/app/misc/Transaction.h>
 #include <ripple/app/misc/TxQ.h>
+#include <ripple/app/tx/impl/Import.h>
+#include <ripple/app/tx/impl/details/NFTokenUtils.h>
 #include <ripple/basics/Log.h>
 #include <ripple/basics/Slice.h>
 #include <ripple/protocol/ErrorCodes.h>
+#include <ripple/protocol/TxFlags.h>
+#include <ripple/protocol/st.h>
 #include <ripple/protocol/tokens.h>
 #include <boost/multiprecision/cpp_dec_float.hpp>
 #include <any>

src/ripple/app/ledger/Ledger.cpp

@@ -29,6 +29,7 @@
 #include <ripple/app/misc/HashRouter.h>
 #include <ripple/app/misc/LoadFeeTrack.h>
 #include <ripple/app/misc/NetworkOPs.h>
+#include <ripple/app/rdb/backend/LMDBDatabase.h>
 #include <ripple/app/rdb/backend/PostgresDatabase.h>
 #include <ripple/app/rdb/backend/SQLiteDatabase.h>
 #include <ripple/basics/Log.h>
@@ -1010,14 +1011,28 @@ saveValidatedLedger(
         return true;
     }
 
-    auto const db = dynamic_cast<SQLiteDatabase*>(&app.getRelationalDatabase());
+    if (app.config().RELATIONAL_DB == 0)
+    {
+        auto const db = dynamic_cast<SQLiteDatabase*>(&app.getRelationalDatabase());
+        if (!db)
+            Throw<std::runtime_error>("Failed to get relational database");
+
+        auto const res = db->saveValidatedLedger(ledger, current);
+        // Clients can now trust the database for
+        // information about this ledger sequence.
+
+        app.pendingSaves().finishWork(seq);
+        return res;
+    }
+
+    auto const db = dynamic_cast<LMDBDatabase*>(&app.getRelationalDatabase());
     if (!db)
         Throw<std::runtime_error>("Failed to get relational database");
 
     auto const res = db->saveValidatedLedger(ledger, current);
-
     // Clients can now trust the database for
     // information about this ledger sequence.
+
     app.pendingSaves().finishWork(seq);
     return res;
 }

src/ripple/app/main/DBInit.h

@@ -41,6 +41,7 @@ inline constexpr std::uint32_t SQLITE_TUNING_CUTOFF = 10'000'000;
 
 // Ledger database holds ledgers and ledger confirmations
 inline constexpr auto LgrDBName{"ledger.db"};
+inline constexpr auto LMDBLgrDBName{"ledger"};
 
 inline constexpr std::array<char const*, 1> LgrDBPragma{
     {"PRAGMA journal_size_limit=1582080;"}};
@@ -71,6 +72,7 @@ inline constexpr std::array<char const*, 5> LgrDBInit{
 
 // Transaction database holds transactions and public keys
 inline constexpr auto TxDBName{"transaction.db"};
+inline constexpr auto LMDBTxDBName{"transaction"};
 
 // In C++17 omitting the explicit template parameters caused
 // a crash
@@ -215,6 +217,7 @@ inline constexpr std::array<char const*, 2> FinalShardDBPragma{
 ////////////////////////////////////////////////////////////////////////////////
 
 inline constexpr auto WalletDBName{"wallet.db"};
+inline constexpr auto LMDBWalletDBName{"wallet"};
 
 inline constexpr std::array<char const*, 6> WalletDBInit{
     {"BEGIN TRANSACTION;",

src/ripple/app/main/NodeIdentity.cpp

@@ -58,12 +58,12 @@ getNodeIdentity(
         return {publicKey, secretKey};
     }
 
-    auto db = app.getWalletDB().checkoutDb();
+    auto db = app.getWalletDB().checkoutLMDB();
 
     if (cmdline.count("newnodeid") != 0)
-        clearNodeIdentity(*db);
+        clearNodeIdentity(db.get());
 
-    return getNodeIdentity(*db);
+    return getNodeIdentity(db.get());
 }
 
 }  // namespace ripple

src/ripple/app/misc/NegativeUNLVote.cpp

@@ -19,7 +19,9 @@
 
 #include <ripple/app/consensus/RCLValidations.h>
 #include <ripple/app/ledger/Ledger.h>
+#include <ripple/app/main/Application.h>
 #include <ripple/app/misc/NegativeUNLVote.h>
+#include <ripple/json/to_string.h>
 
 namespace ripple {
 

src/ripple/app/misc/NetworkOPs.cpp

@@ -38,6 +38,7 @@
 #include <ripple/app/misc/ValidatorKeys.h>
 #include <ripple/app/misc/ValidatorList.h>
 #include <ripple/app/misc/impl/AccountTxPaging.h>
+#include <ripple/app/rdb/backend/LMDBDatabase.h>
 #include <ripple/app/rdb/backend/PostgresDatabase.h>
 #include <ripple/app/rdb/backend/SQLiteDatabase.h>
 #include <ripple/app/reporting/ReportingETL.h>
@@ -3542,7 +3543,7 @@ NetworkOPsImp::unsubAccountInternal(
 void
 NetworkOPsImp::addAccountHistoryJob(SubAccountHistoryInfoWeak subInfo)
 {
-    enum DatabaseType { Postgres, Sqlite, None };
+    enum DatabaseType { Postgres, Sqlite, LMDB, None };
     static const auto databaseType = [&]() -> DatabaseType {
 #ifdef RIPPLED_REPORTING
         if (app_.config().reporting())

src/ripple/app/misc/SHAMapStoreImp.cpp

@@ -22,6 +22,7 @@
 #include <ripple/app/ledger/TransactionMaster.h>
 #include <ripple/app/misc/NetworkOPs.h>
 #include <ripple/app/rdb/State.h>
+#include <ripple/app/rdb/backend/LMDBDatabase.h>
 #include <ripple/app/rdb/backend/SQLiteDatabase.h>
 #include <ripple/beast/core/CurrentThreadName.h>
 #include <ripple/core/ConfigSections.h>
@@ -634,8 +635,53 @@ SHAMapStoreImp::clearPrior(LedgerIndex lastRotated)
     if (healthWait() == stopping)
         return;
 
-    SQLiteDatabase* const db =
-        dynamic_cast<SQLiteDatabase*>(&app_.getRelationalDatabase());
+    if (app_.config().RELATIONAL_DB == 0)
+    {
+
+        SQLiteDatabase* const db =
+            dynamic_cast<SQLiteDatabase*>(&app_.getRelationalDatabase());
+
+        if (!db)
+            Throw<std::runtime_error>("Failed to get relational database");
+
+        clearSql(
+            lastRotated,
+            "Ledgers",
+            [db]() -> std::optional<LedgerIndex> { return db->getMinLedgerSeq(); },
+            [db](LedgerIndex min) -> void { db->deleteBeforeLedgerSeq(min); });
+        if (healthWait() == stopping)
+            return;
+
+        if (!app_.config().useTxTables())
+            return;
+
+        clearSql(
+            lastRotated,
+            "Transactions",
+            [&db]() -> std::optional<LedgerIndex> {
+                return db->getTransactionsMinLedgerSeq();
+            },
+            [&db](LedgerIndex min) -> void {
+                db->deleteTransactionsBeforeLedgerSeq(min);
+            });
+        if (healthWait() == stopping)
+            return;
+
+        clearSql(
+            lastRotated,
+            "AccountTransactions",
+            [&db]() -> std::optional<LedgerIndex> {
+                return db->getAccountTransactionsMinLedgerSeq();
+            },
+            [&db](LedgerIndex min) -> void {
+                db->deleteAccountTransactionsBeforeLedgerSeq(min);
+            });
+        if (healthWait() == stopping)
+            return;
+    }
+
+    LMDBDatabase* const db =
+        dynamic_cast<LMDBDatabase*>(&app_.getRelationalDatabase());
 
     if (!db)
         Throw<std::runtime_error>("Failed to get relational database");

src/ripple/app/misc/impl/AmendmentTable.cpp

@@ -328,8 +328,8 @@ AmendmentTableImpl::AmendmentTableImpl(
 
     // Find out if the FeatureVotes table exists in WalletDB
     bool const featureVotesExist = [this]() {
-        auto db = db_.checkoutDb();
-        return createFeatureVotes(*db);
+        auto db = db_.checkoutLMDB();
+        return createFeatureVotes(db.get());
     }();
 
     // Parse supported amendments
@@ -404,9 +404,9 @@ AmendmentTableImpl::AmendmentTableImpl(
     }
 
     // Read amendment votes from wallet.db
-    auto db = db_.checkoutDb();
+    auto db = db_.checkoutLMDB();
     readAmendments(
-        *db,
+        db.get(),
         [&](boost::optional<std::string> amendment_hash,
             boost::optional<std::string> amendment_name,
             boost::optional<AmendmentVote> vote) {
@@ -506,8 +506,8 @@ AmendmentTableImpl::persistVote(
     AmendmentVote vote) const
 {
     assert(vote != AmendmentVote::obsolete);
-    auto db = db_.checkoutDb();
-    voteAmendment(*db, amendment, name, vote);
+    auto db = db_.checkoutLMDB();
+    voteAmendment(db.get(), amendment, name, vote);
 }
 
 bool

src/ripple/app/misc/impl/Manifest.cpp

@@ -512,8 +512,8 @@ ManifestCache::applyManifest(Manifest m)
 void
 ManifestCache::load(DatabaseCon& dbCon, std::string const& dbTable)
 {
-    auto db = dbCon.checkoutDb();
-    ripple::getManifests(*db, dbTable, *this, j_);
+    auto db = dbCon.checkoutLMDB();
+    ripple::getManifests(db.get(), dbTable, *this, j_);
 }
 
 bool
@@ -580,8 +580,8 @@ ManifestCache::save(
     std::function<bool(PublicKey const&)> const& isTrusted)
 {
     std::shared_lock lock{mutex_};
-    auto db = dbCon.checkoutDb();
+    auto db = dbCon.checkoutLMDB();
 
-    saveManifests(*db, dbTable, isTrusted, map_, j_);
+    saveManifests(db.get(), dbTable, isTrusted, map_, j_);
 }
 }  // namespace ripple

src/ripple/app/misc/impl/Transaction.cpp

@@ -21,6 +21,7 @@
 #include <ripple/app/main/Application.h>
 #include <ripple/app/misc/HashRouter.h>
 #include <ripple/app/misc/Transaction.h>
+#include <ripple/app/rdb/backend/LMDBDatabase.h>
 #include <ripple/app/rdb/backend/PostgresDatabase.h>
 #include <ripple/app/rdb/backend/SQLiteDatabase.h>
 #include <ripple/app/tx/apply.h>
@@ -162,7 +163,20 @@ Transaction::load(
     std::optional<ClosedInterval<uint32_t>> const& range,
     error_code_i& ec)
 {
-    auto const db = dynamic_cast<SQLiteDatabase*>(&app.getRelationalDatabase());
+
+    if (app.config().RELATIONAL_DB == 0)
+    {
+        auto const db = dynamic_cast<SQLiteDatabase*>(&app.getRelationalDatabase());
+
+        if (!db)
+        {
+            Throw<std::runtime_error>("Failed to get relational database");
+        }
+
+        return db->getTransaction(id, range, ec);
+    }
+
+     auto const db = dynamic_cast<LMDBDatabase*>(&app.getRelationalDatabase());
 
     if (!db)
     {