[WIP] full-chain membership proof++ integration

.github/workflows/build.yml

@@ -68,7 +68,7 @@ jobs:
     - uses: msys2/setup-msys2@v2
       with:
         update: true
-        install: mingw-w64-x86_64-toolchain make mingw-w64-x86_64-cmake mingw-w64-x86_64-ccache mingw-w64-x86_64-boost mingw-w64-x86_64-openssl mingw-w64-x86_64-zeromq mingw-w64-x86_64-libsodium mingw-w64-x86_64-hidapi mingw-w64-x86_64-libusb mingw-w64-x86_64-unbound git
+        install: mingw-w64-x86_64-toolchain make mingw-w64-x86_64-cmake mingw-w64-x86_64-ccache mingw-w64-x86_64-boost mingw-w64-x86_64-openssl mingw-w64-x86_64-zeromq mingw-w64-x86_64-libsodium mingw-w64-x86_64-hidapi mingw-w64-x86_64-libusb mingw-w64-x86_64-unbound git mingw-w64-x86_64-rust
     - shell: msys2 {0}
       run: |
            curl -O https://repo.msys2.org/mingw/mingw64/mingw-w64-x86_64-protobuf-c-1.4.1-1-any.pkg.tar.zst

.github/workflows/depends.yml

@@ -31,36 +31,47 @@ jobs:
         toolchain:
           - name: "RISCV 64bit"
             host: "riscv64-linux-gnu"
+            rust_host: "riscv64gc-unknown-linux-gnu"
             packages: "python3 gperf g++-riscv64-linux-gnu"
           - name: "ARM v7"
             host: "arm-linux-gnueabihf"
+            rust_host: "armv7-unknown-linux-gnueabihf"
             packages: "python3 gperf g++-arm-linux-gnueabihf"
           - name: "ARM v8"
             host: "aarch64-linux-gnu"
+            rust_host: "aarch64-unknown-linux-gnu"
             packages: "python3 gperf g++-aarch64-linux-gnu"
           - name: "i686 Win"
             host: "i686-w64-mingw32"
+            rust_host: "i686-pc-windows-gnu"
             packages: "python3 g++-mingw-w64-i686"
           - name: "i686 Linux"
             host: "i686-pc-linux-gnu"
+            rust_host: "i686-unknown-linux-gnu"
             packages: "gperf cmake g++-multilib python3-zmq"
           - name: "Win64"
             host: "x86_64-w64-mingw32"
+            rust_host: "x86_64-pc-windows-gnu"
             packages: "cmake python3 g++-mingw-w64-x86-64"
           - name: "x86_64 Linux"
             host: "x86_64-unknown-linux-gnu"
+            rust_host: "x86_64-unknown-linux-gnu"
             packages: "gperf cmake python3-zmq libdbus-1-dev libharfbuzz-dev"
           - name: "Cross-Mac x86_64"
             host: "x86_64-apple-darwin"
+            rust_host: "x86_64-apple-darwin"
             packages: "cmake imagemagick libcap-dev librsvg2-bin libz-dev libbz2-dev libtiff-tools python-dev python3-setuptools-git"
           - name: "Cross-Mac aarch64"
             host: "aarch64-apple-darwin"
+            rust_host: "aarch64-apple-darwin"
             packages: "cmake imagemagick libcap-dev librsvg2-bin libz-dev libbz2-dev libtiff-tools python-dev python3-setuptools-git"
           - name: "x86_64 Freebsd"
             host: "x86_64-unknown-freebsd"
+            rust_host: "x86_64-unknown-freebsd"
             packages: "clang-8 gperf cmake python3-zmq libdbus-1-dev libharfbuzz-dev"
           - name: "ARMv8 Android"
             host: "aarch64-linux-android"
+            rust_host: "aarch64-linux-android"
             packages: "gperf cmake python3"
     name: ${{ matrix.toolchain.name }}
     steps:
@@ -95,6 +106,11 @@ jobs:
       run: ${{env.APT_SET_CONF}}
     - name: install dependencies
       run: sudo apt update; sudo apt -y install build-essential libtool cmake autotools-dev automake pkg-config bsdmainutils curl git ca-certificates ccache ${{ matrix.toolchain.packages }}
+    - name: install rust target
+      # We can't use the latest Rust due to LLVM 17 not working with old `ld`s (such as in Ubuntu 20.04) for RISC-V
+      # We could update ld (a pain), update Ubuntu (requires a large amount of changes), or downgrade Rust
+      # We can't use Rust 1.70 due to LLVM 16 requiring ld >= 2.40 when building for Windows
+      run: rustup toolchain install 1.69; rustup default 1.69; rustup target add ${{ matrix.toolchain.rust_host }}
     - name: prepare w64-mingw32
       if: ${{ matrix.toolchain.host == 'x86_64-w64-mingw32' || matrix.toolchain.host == 'i686-w64-mingw32' }}
       run: |

src/CMakeLists.txt

@@ -89,6 +89,7 @@ add_subdirectory(ringct)
 add_subdirectory(checkpoints)
 add_subdirectory(cryptonote_basic)
 add_subdirectory(cryptonote_core)
+add_subdirectory(fcmp_pp)
 add_subdirectory(lmdb)
 add_subdirectory(multisig)
 add_subdirectory(net)

src/blockchain_db/CMakeLists.txt

@@ -45,6 +45,7 @@ target_link_libraries(blockchain_db
   PUBLIC
     common
     cncrypto
+    fcmp_pp
     ringct
     ${LMDB_LIBRARY}
     ${Boost_FILESYSTEM_LIBRARY}

src/blockchain_db/blockchain_db.cpp

@@ -41,6 +41,60 @@
 
 using epee::string_tools::pod_to_hex;
 
+//---------------------------------------------------------------
+// Helper function to group outputs by unlock block
+static void get_outs_by_unlock_block(const cryptonote::transaction &tx,
+  const std::vector<uint64_t> &output_ids,
+  const uint64_t tx_height,
+  const bool miner_tx,
+  fcmp_pp::curve_trees::OutputsByUnlockBlock &outs_by_unlock_block_inout)
+{
+  const uint64_t unlock_block = cryptonote::get_unlock_block_index(tx.unlock_time, tx_height);
+
+  CHECK_AND_ASSERT_THROW_MES(tx.vout.size() == output_ids.size(), "unexpected size of output ids");
+
+  for (std::size_t i = 0; i < tx.vout.size(); ++i)
+  {
+    const auto &out = tx.vout[i];
+
+    crypto::public_key output_public_key;
+    if (!cryptonote::get_output_public_key(out, output_public_key))
+      throw std::runtime_error("Could not get an output public key from a tx output.");
+
+    static_assert(CURRENT_TRANSACTION_VERSION == 2, "This section of code was written with 2 tx versions in mind. "
+      "Revisit this section and update for the new tx version.");
+    CHECK_AND_ASSERT_THROW_MES(tx.version == 1 || tx.version == 2, "encountered unexpected tx version");
+
+    if (!miner_tx && tx.version == 2)
+      CHECK_AND_ASSERT_THROW_MES(tx.rct_signatures.outPk.size() > i, "unexpected size of outPk");
+
+    rct::key commitment = (miner_tx || tx.version != 2)
+      ? rct::zeroCommit(out.amount)
+      : tx.rct_signatures.outPk[i].mask;
+
+    auto output_pair = fcmp_pp::curve_trees::OutputPair{
+        .output_pubkey = std::move(output_public_key),
+        .commitment    = std::move(commitment)
+      };
+
+    auto output_context = fcmp_pp::curve_trees::OutputContext{
+        .output_id   = output_ids[i],
+        .output_pair = std::move(output_pair)
+      };
+
+    if (outs_by_unlock_block_inout.find(unlock_block) == outs_by_unlock_block_inout.end())
+    {
+      auto new_vec = std::vector<fcmp_pp::curve_trees::OutputContext>{std::move(output_context)};
+      outs_by_unlock_block_inout[unlock_block] = std::move(new_vec);
+    }
+    else
+    {
+      outs_by_unlock_block_inout[unlock_block].emplace_back(std::move(output_context));
+    }
+  }
+}
+//---------------------------------------------------------------
+
 namespace cryptonote
 {
 
@@ -179,7 +233,7 @@ void BlockchainDB::pop_block()
   pop_block(blk, txs);
 }
 
-void BlockchainDB::add_transaction(const crypto::hash& blk_hash, const std::pair<transaction, blobdata_ref>& txp, const crypto::hash* tx_hash_ptr, const crypto::hash* tx_prunable_hash_ptr)
+std::vector<uint64_t> BlockchainDB::add_transaction(const crypto::hash& blk_hash, const std::pair<transaction, blobdata_ref>& txp, const crypto::hash* tx_hash_ptr, const crypto::hash* tx_prunable_hash_ptr)
 {
   const transaction &tx = txp.first;
 
@@ -223,29 +277,43 @@ void BlockchainDB::add_transaction(const crypto::hash& blk_hash, const std::pair
 
   uint64_t tx_id = add_transaction_data(blk_hash, txp, tx_hash, tx_prunable_hash);
 
-  std::vector<uint64_t> amount_output_indices(tx.vout.size());
+  std::vector<output_indexes_t> output_indices(tx.vout.size());
 
   // iterate tx.vout using indices instead of C++11 foreach syntax because
   // we need the index
   for (uint64_t i = 0; i < tx.vout.size(); ++i)
   {
     // miner v2 txes have their coinbase output in one single out to save space,
     // and we store them as rct outputs with an identity mask
+    // note: get_outs_by_unlock_block mirrors this logic
     if (miner_tx && tx.version == 2)
     {
       cryptonote::tx_out vout = tx.vout[i];
       rct::key commitment = rct::zeroCommit(vout.amount);
       vout.amount = 0;
-      amount_output_indices[i] = add_output(tx_hash, vout, i, tx.unlock_time,
+      output_indices[i] = add_output(tx_hash, vout, i, tx.unlock_time,
         &commitment);
     }
     else
     {
-      amount_output_indices[i] = add_output(tx_hash, tx.vout[i], i, tx.unlock_time,
+      output_indices[i] = add_output(tx_hash, tx.vout[i], i, tx.unlock_time,
         tx.version > 1 ? &tx.rct_signatures.outPk[i].mask : NULL);
     }
   }
+
+  std::vector<uint64_t> amount_output_indices;
+  std::vector<uint64_t> output_ids;
+  amount_output_indices.reserve(output_indices.size());
+  output_ids.reserve(output_indices.size());
+  for (const auto &o_idx : output_indices)
+  {
+    amount_output_indices.push_back(o_idx.amount_index);
+    output_ids.push_back(o_idx.output_id);
+  }
+
   add_tx_amount_output_indices(tx_id, amount_output_indices);
+
+  return output_ids;
 }
 
 uint64_t BlockchainDB::add_block( const std::pair<block, blobdata>& blck
@@ -273,17 +341,20 @@ uint64_t BlockchainDB::add_block( const std::pair<block, blobdata>& blck
 
   time1 = epee::misc_utils::get_tick_count();
 
+  std::vector<std::vector<uint64_t>> output_ids;
+  output_ids.reserve(txs.size());
+
   uint64_t num_rct_outs = 0;
   blobdata miner_bd = tx_to_blob(blk.miner_tx);
-  add_transaction(blk_hash, std::make_pair(blk.miner_tx, blobdata_ref(miner_bd)));
+  std::vector<uint64_t> miner_output_ids = add_transaction(blk_hash, std::make_pair(blk.miner_tx, blobdata_ref(miner_bd)));
   if (blk.miner_tx.version == 2)
     num_rct_outs += blk.miner_tx.vout.size();
   int tx_i = 0;
   crypto::hash tx_hash = crypto::null_hash;
   for (const std::pair<transaction, blobdata>& tx : txs)
   {
     tx_hash = blk.tx_hashes[tx_i];
-    add_transaction(blk_hash, tx, &tx_hash);
+    output_ids.push_back(add_transaction(blk_hash, tx, &tx_hash));
     for (const auto &vout: tx.first.vout)
     {
       if (vout.amount == 0)
@@ -294,9 +365,32 @@ uint64_t BlockchainDB::add_block( const std::pair<block, blobdata>& blck
   TIME_MEASURE_FINISH(time1);
   time_add_transaction += time1;
 
+  // When adding a block, we also need to keep track of when outputs unlock, so
+  // we can use them to grow the merkle tree used in fcmp's at that point.
+  fcmp_pp::curve_trees::OutputsByUnlockBlock outs_by_unlock_block;
+
+  // Get miner tx's leaf tuples
+  get_outs_by_unlock_block(
+    blk.miner_tx,
+    miner_output_ids,
+    prev_height,
+    true/*miner_tx*/,
+    outs_by_unlock_block);
+
+  // Get all other txs' leaf tuples
+  for (std::size_t i = 0; i < txs.size(); ++i)
+  {
+    get_outs_by_unlock_block(
+      txs[i].first,
+      output_ids[i],
+      prev_height,
+      false/*miner_tx*/,
+      outs_by_unlock_block);
+  }
+
   // call out to subclass implementation to add the block & metadata
   time1 = epee::misc_utils::get_tick_count();
-  add_block(blk, block_weight, long_term_block_weight, cumulative_difficulty, coins_generated, num_rct_outs, blk_hash);
+  add_block(blk, block_weight, long_term_block_weight, cumulative_difficulty, coins_generated, num_rct_outs, blk_hash, outs_by_unlock_block);
   TIME_MEASURE_FINISH(time1);
   time_add_block1 += time1;
 

src/blockchain_db/blockchain_db.h

@@ -32,6 +32,8 @@
 
 #include <string>
 #include <exception>
+#include <map>
+#include <memory>
 #include <boost/program_options.hpp>
 #include "common/command_line.h"
 #include "crypto/hash.h"
@@ -40,6 +42,7 @@
 #include "cryptonote_basic/difficulty.h"
 #include "cryptonote_basic/hardfork.h"
 #include "cryptonote_protocol/enums.h"
+#include "fcmp_pp/curve_trees.h"
 
 /** \file
  * Cryptonote Blockchain Database Interface
@@ -187,6 +190,14 @@ struct txpool_tx_meta_t
   }
 };
 
+/**
+ * @brief a struct containing output indexes for convenience
+ */
+struct output_indexes_t
+{
+  uint64_t amount_index;
+  uint64_t output_id;
+};
 
 #define DBF_SAFE       1
 #define DBF_FAST       2
@@ -398,6 +409,7 @@ class BlockchainDB
    * @param cumulative_difficulty the accumulated difficulty after this block
    * @param coins_generated the number of coins generated total after this block
    * @param blk_hash the hash of the block
+   * @param outs_by_unlock_block the outputs from this block to add to the merkle tree
    */
   virtual void add_block( const block& blk
                 , size_t block_weight
@@ -406,6 +418,7 @@ class BlockchainDB
                 , const uint64_t& coins_generated
                 , uint64_t num_rct_outs
                 , const crypto::hash& blk_hash
+                , const fcmp_pp::curve_trees::OutputsByUnlockBlock& outs_by_unlock_block
                 ) = 0;
 
   /**
@@ -470,8 +483,9 @@ class BlockchainDB
    * future, this tracking (of the number, at least) should be moved to
    * this class, as it is necessary and the same among all BlockchainDB.
    *
-   * It returns an amount output index, which is the index of the output
-   * for its specified amount.
+   * It returns the output indexes, which contains an amount output index (the
+   * index of the output for its specified amount) and output id (the global
+   * index of the output among all outputs of any amount).
    *
    * This data should be stored in such a manner that the only thing needed to
    * reverse the process is the tx_out.
@@ -484,9 +498,9 @@ class BlockchainDB
    * @param local_index index of the output in its transaction
    * @param unlock_time unlock time/height of the output
    * @param commitment the rct commitment to the output amount
-   * @return amount output index
+   * @return output indexes
    */
-  virtual uint64_t add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index, const uint64_t unlock_time, const rct::key *commitment) = 0;
+  virtual output_indexes_t add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index, const uint64_t unlock_time, const rct::key *commitment) = 0;
 
   /**
    * @brief store amount output indices for a tx's outputs
@@ -567,21 +581,25 @@ class BlockchainDB
    * @param tx the transaction to add
    * @param tx_hash_ptr the hash of the transaction, if already calculated
    * @param tx_prunable_hash_ptr the hash of the prunable part of the transaction, if already calculated
+   *
+   * @return the global output ids of all outputs inserted
    */
-  void add_transaction(const crypto::hash& blk_hash, const std::pair<transaction, blobdata_ref>& tx, const crypto::hash* tx_hash_ptr = NULL, const crypto::hash* tx_prunable_hash_ptr = NULL);
+  std::vector<uint64_t> add_transaction(const crypto::hash& blk_hash, const std::pair<transaction, blobdata_ref>& tx, const crypto::hash* tx_hash_ptr = NULL, const crypto::hash* tx_prunable_hash_ptr = NULL);
 
   mutable uint64_t time_tx_exists = 0;  //!< a performance metric
   uint64_t time_commit1 = 0;  //!< a performance metric
   bool m_auto_remove_logs = true;  //!< whether or not to automatically remove old logs
 
   HardFork* m_hardfork;
 
+  std::shared_ptr<fcmp_pp::curve_trees::CurveTreesV1> m_curve_trees;
+
 public:
 
   /**
    * @brief An empty constructor.
    */
-  BlockchainDB(): m_hardfork(NULL), m_open(false) { }
+  BlockchainDB(): m_hardfork(NULL), m_open(false), m_curve_trees() { }
 
   /**
    * @brief An empty destructor.
@@ -1764,6 +1782,15 @@ class BlockchainDB
    */
   virtual bool for_all_alt_blocks(std::function<bool(const crypto::hash &blkid, const alt_block_data_t &data, const cryptonote::blobdata_ref *blob)> f, bool include_blob = false) const = 0;
 
+  // TODO: description and make private
+  virtual void grow_tree(std::vector<fcmp_pp::curve_trees::OutputContext> &&new_outputs) = 0;
+
+  virtual void trim_tree(const uint64_t trim_n_leaf_tuples) = 0;
+
+  // TODO: description
+  virtual bool audit_tree(const uint64_t expected_n_leaf_tuples) const = 0;
+  virtual uint64_t get_num_leaf_tuples() const = 0;
+  virtual std::array<uint8_t, 32UL> get_tree_root() const = 0;
 
   //
   // Hard fork related storage