Merge remote-tracking branch 'origin/hotstuff_integration' into GH-2102…

…-vote-processing

.cicd/platforms/asan.Dockerfile

@@ -0,0 +1,37 @@
+# syntax=docker/dockerfile:1
+FROM ubuntu:jammy
+ENV TZ="America/New_York"
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt-get update && apt-get upgrade -y && \
+    apt-get install -y build-essential      \
+                       cmake                \
+                       git                  \
+                       jq                   \
+                       libcurl4-openssl-dev \
+                       libgmp-dev           \
+                       llvm-11-dev          \
+                       lsb-release          \
+                       ninja-build          \
+                       python3-numpy        \
+                       software-properties-common \
+                       file                 \
+                       wget                 \
+                       zlib1g-dev           \
+                       zstd
+
+RUN yes | bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" llvm.sh 18
+
+#make sure no confusion on what llvm library leap's cmake should pick up on
+RUN rm -rf /usr/lib/llvm-18/lib/cmake
+
+ENV LEAP_PLATFORM_HAS_EXTRAS_CMAKE=1
+COPY <<-EOF /extras.cmake
+  set(CMAKE_BUILD_TYPE "RelWithDebInfo" CACHE STRING "" FORCE)
+
+  set(CMAKE_C_COMPILER "clang-18" CACHE STRING "")
+  set(CMAKE_CXX_COMPILER "clang++-18" CACHE STRING "")
+  set(CMAKE_C_FLAGS "-fsanitize=address -fno-omit-frame-pointer" CACHE STRING "")
+  set(CMAKE_CXX_FLAGS "-fsanitize=address -fno-omit-frame-pointer" CACHE STRING "")
+EOF
+
+ENV ASAN_OPTIONS=detect_leaks=0

.github/workflows/build.yaml

@@ -137,6 +137,7 @@ jobs:
           - cfg: {name: 'ubuntu22',      base: 'ubuntu22', builddir: 'ubuntu22'}
           - cfg: {name: 'asserton',      base: 'asserton', builddir: 'asserton'}
           - cfg: {name: 'ubsan',         base: 'ubsan',    builddir: 'ubsan'}
+          - cfg: {name: 'asan',          base: 'asan',     builddir: 'asan'}
           - cfg: {name: 'ubuntu20repro', base: 'ubuntu20', builddir: 'reproducible'}
           - cfg: {name: 'ubuntu22repro', base: 'ubuntu22', builddir: 'reproducible'}
     runs-on: ["self-hosted", "enf-x86-hightier"]

benchmark/benchmark.cpp

@@ -15,7 +15,8 @@ std::map<std::string, std::function<void()>> features {
    { "key", key_benchmarking },
    { "hash", hash_benchmarking },
    { "blake2", blake2_benchmarking },
-   { "bls", bls_benchmarking }
+   { "bls", bls_benchmarking },
+   { "merkle", merkle_benchmarking }
 };
 
 // values to control cout format
@@ -34,6 +35,10 @@ void set_num_runs(uint32_t runs) {
    num_runs = runs;
 }
 
+uint32_t get_num_runs() {
+   return num_runs;
+}
+
 void print_header() {
    std::cout << std::left << std::setw(name_width) << "function"
       << std::setw(runs_width) << "runs"
@@ -63,12 +68,14 @@ bytes to_bytes(const std::string& source) {
    return output;
 };
 
-void benchmarking(const std::string& name, const std::function<void()>& func) {
+void benchmarking(const std::string& name, const std::function<void()>& func,
+                  std::optional<size_t> opt_num_runs /* = {} */) {
    uint64_t total{0};
    uint64_t min{std::numeric_limits<uint64_t>::max()};
    uint64_t max{0};
+   uint32_t runs = opt_num_runs ? *opt_num_runs : num_runs;
 
-   for (auto i = 0U; i < num_runs; ++i) {
+   for (auto i = 0U; i < runs; ++i) {
       auto start_time = std::chrono::high_resolution_clock::now();
       func();
       auto end_time = std::chrono::high_resolution_clock::now();
@@ -79,7 +86,7 @@ void benchmarking(const std::string& name, const std::function<void()>& func) {
       max = std::max(max, duration);
    }
 
-   print_results(name, num_runs, total, min, max);
+   print_results(name, runs, total, min, max);
 }
 
 } // benchmark

benchmark/benchmark.hpp

@@ -4,13 +4,15 @@
 #include <map>
 #include <vector>
 #include <limits>
+#include <optional>
 
 #include <fc/crypto/hex.hpp>
 
 namespace eosio::benchmark {
 using bytes = std::vector<char>;
 
 void set_num_runs(uint32_t runs);
+uint32_t get_num_runs();
 std::map<std::string, std::function<void()>> get_features();
 void print_header();
 bytes to_bytes(const std::string& source);
@@ -21,7 +23,8 @@ void key_benchmarking();
 void hash_benchmarking();
 void blake2_benchmarking();
 void bls_benchmarking();
+void merkle_benchmarking();
 
-void benchmarking(const std::string& name, const std::function<void()>& func); 
+void benchmarking(const std::string& name, const std::function<void()>& func, std::optional<size_t> num_runs = {});
 
 } // benchmark

benchmark/bls.cpp

@@ -153,7 +153,7 @@ void benchmark_bls_g1_add() {
    // prepare g1 operand in Jacobian LE format
    g1 p = random_g1();
    std::array<char, 96> op;
-   p.toAffineBytesLE(std::span<uint8_t, 96>((uint8_t*)op.data(), 96), false);
+   p.toAffineBytesLE(std::span<uint8_t, 96>((uint8_t*)op.data(), 96), from_mont::yes);
 
    // prepare result operand
    std::array<char, 96> result;
@@ -172,7 +172,7 @@ void benchmark_bls_g2_add() {
    // prepare g2 operand in Jacobian LE format
    g2 p = random_g2();
    std::array<char, 192> op;
-   p.toAffineBytesLE(std::span<uint8_t, 192>((uint8_t*)op.data(), 192), false);
+   p.toAffineBytesLE(std::span<uint8_t, 192>((uint8_t*)op.data(), 192), from_mont::yes);
 
    // prepare result operand
    std::array<char, 192> result;
@@ -192,7 +192,7 @@ void benchmark_bls_g1_weighted_sum_impl(const std::string& test_name, uint32_t n
    std::vector<char> g1_buf(96*num_points);
    for (auto i=0u; i < num_points; ++i) {
       g1 p = random_g1();
-      p.toAffineBytesLE(std::span<uint8_t, 96>((uint8_t*)g1_buf.data() + i * 96, 96), false);
+      p.toAffineBytesLE(std::span<uint8_t, 96>((uint8_t*)g1_buf.data() + i * 96, 96), from_mont::yes);
    }
    chain::span<const char> g1_points(g1_buf.data(), g1_buf.size());
 
@@ -237,7 +237,7 @@ void benchmark_bls_g2_weighted_sum_impl(const std::string& test_name, uint32_t n
    std::vector<char> g2_buf(192*num_points);
    for (auto i=0u; i < num_points; ++i) {
       g2 p = random_g2();
-      p.toAffineBytesLE(std::span<uint8_t, 192>((uint8_t*)g2_buf.data() + i * 192, 192), false);
+      p.toAffineBytesLE(std::span<uint8_t, 192>((uint8_t*)g2_buf.data() + i * 192, 192), from_mont::yes);
    }
    eosio::chain::span<const char> g2_points(g2_buf.data(), g2_buf.size());
 
@@ -282,15 +282,15 @@ void benchmark_bls_pairing_impl(const std::string& test_name, uint32_t num_pairs
    std::vector<char> g1_buf(96*num_pairs);
    for (auto i=0u; i < num_pairs; ++i) {
       g1 p = random_g1();
-      p.toAffineBytesLE(std::span<uint8_t, 96>((uint8_t*)g1_buf.data() + i * 96, 96), false);
+      p.toAffineBytesLE(std::span<uint8_t, 96>((uint8_t*)g1_buf.data() + i * 96, 96), from_mont::yes);
    }
    eosio::chain::span<const char> g1_points(g1_buf.data(), g1_buf.size());
 
    // prepare g2 operand
    std::vector<char> g2_buf(192*num_pairs);
    for (auto i=0u; i < num_pairs; ++i) {
       g2 p2 = random_g2();
-      p2.toAffineBytesLE(std::span<uint8_t, (192)>((uint8_t*)g2_buf.data() + i * 192, (192)), false);
+      p2.toAffineBytesLE(std::span<uint8_t, (192)>((uint8_t*)g2_buf.data() + i * 192, (192)), from_mont::yes);
    }
    eosio::chain::span<const char> g2_points(g2_buf.data(), g2_buf.size());
 
@@ -321,7 +321,7 @@ void benchmark_bls_g1_map() {
    // prepare e operand. Must be fp LE.
    std::array<char, 48> e;
    fp a = random_fe();
-   a.toBytesLE(std::span<uint8_t, 48>((uint8_t*)e.data(), 48), false);
+   a.toBytesLE(std::span<uint8_t, 48>((uint8_t*)e.data(), 48), from_mont::yes);
 
    // prepare result operand
    std::array<char, 96> result;
@@ -339,7 +339,7 @@ void benchmark_bls_g1_map() {
 void benchmark_bls_g2_map() {
    std::array<char, 96> e;
    fp2 a = random_fe2();
-   a.toBytesLE(std::span<uint8_t, 96>((uint8_t*)e.data(), 96), false);
+   a.toBytesLE(std::span<uint8_t, 96>((uint8_t*)e.data(), 96), from_mont::yes);
 
    // prepare result operand
    std::array<char, 192> result;
@@ -379,12 +379,12 @@ void benchmark_bls_fp_mul() {
    // prepare op1
    std::array<char, 48> op1;
    fp a = random_fe();
-   a.toBytesLE(std::span<uint8_t, 48>((uint8_t*)op1.data(), 48), false);
+   a.toBytesLE(std::span<uint8_t, 48>((uint8_t*)op1.data(), 48), from_mont::yes);
 
    // prepare op2
    std::array<char, 48> op2;
    fp b = random_fe();
-   b.toBytesLE(std::span<uint8_t, 48>((uint8_t*)op2.data(), 48), false);
+   b.toBytesLE(std::span<uint8_t, 48>((uint8_t*)op2.data(), 48), from_mont::yes);
 
    // prepare result operand
    std::array<char, 48> result;
@@ -402,7 +402,7 @@ void benchmark_bls_fp_exp() {
    // prepare base
    std::array<char, 48> base;
    fp a = random_fe();
-   a.toBytesLE(std::span<uint8_t, 48>((uint8_t*)base.data(), 48), false);
+   a.toBytesLE(std::span<uint8_t, 48>((uint8_t*)base.data(), 48), from_mont::yes);
 
    // prepare exp operand
    std::array<char, 64> exp;

benchmark/merkle.cpp

@@ -0,0 +1,69 @@
+#include <benchmark.hpp>
+#include <eosio/chain/incremental_merkle.hpp>
+#include <eosio/chain/incremental_merkle_legacy.hpp>
+#include <random>
+
+namespace eosio::benchmark {
+
+using namespace eosio::chain;
+
+std::vector<digest_type> create_test_digests(size_t n) {
+   std::vector<digest_type> v;
+   v.reserve(n);
+   for (size_t i=0; i<n; ++i)
+      v.push_back(fc::sha256::hash(std::string{"Node"} + std::to_string(i)));
+   return v;
+}
+
+void benchmark_calc_merkle(uint32_t size_boost) {
+   using namespace std::string_literals;
+   const size_t num_digests = size_boost * 1000ull; // don't use exact powers of 2 as it is a special case
+
+   const std::vector<digest_type> digests = create_test_digests(num_digests);
+   const deque<digest_type> deq { digests.begin(), digests.end() };
+
+   auto num_str = std::to_string(size_boost);
+   while(num_str.size() < 4)
+      num_str.insert(0, 1, ' ');
+   auto msg_header = "Calc, "s + num_str + ",000 digests,  "s;
+   uint32_t num_runs = std::min(get_num_runs(), std::max(1u, get_num_runs() / size_boost));
+   benchmarking(msg_header + "legacy: ", [&]() { calculate_merkle_legacy(deq); }, num_runs);
+   benchmarking(msg_header + "savanna:", [&]() { calculate_merkle(digests.begin(), digests.end()); }, num_runs);
+}
+
+void benchmark_incr_merkle(uint32_t size_boost) {
+   using namespace std::string_literals;
+   const size_t num_digests = size_boost * 1000ull; // don't use exact powers of 2 as it is a special case
+
+   const std::vector<digest_type> digests = create_test_digests(num_digests);
+
+   auto num_str = std::to_string(size_boost);
+   while(num_str.size() < 4)
+      num_str.insert(0, 1, ' ');
+   auto msg_header = "Incr, "s + num_str + ",000 digests,  "s;
+   uint32_t num_runs = std::min(get_num_runs(), std::max(1u, get_num_runs() / size_boost));
+
+   auto incr = [&](const auto& incr_tree) {
+      auto work_tree = incr_tree;
+      for (const auto& d : digests)
+         work_tree.append(d);
+      return work_tree.get_root();
+   };
+
+   benchmarking(msg_header + "legacy: ", [&]() { incr(incremental_merkle_tree_legacy()); }, num_runs);
+   benchmarking(msg_header + "savanna:", [&]() { incr(incremental_merkle_tree()); }, num_runs);
+}
+
+// register benchmarking functions
+void merkle_benchmarking() {
+   benchmark_calc_merkle(1000); // calculate_merkle of very large sequence (1,000,000 digests)
+   benchmark_calc_merkle(50);   // calculate_merkle of large sequence (50,000 digests)
+   benchmark_calc_merkle(1);    // calculate_merkle of small sequence (1000 digests)
+   std::cout << "\n";
+
+   benchmark_incr_merkle(100);  // incremental_merkle of very large sequence (100,000 digests)
+   benchmark_incr_merkle(25);   // incremental_merkle of large sequence (25,000 digests)
+   benchmark_incr_merkle(1);    // incremental_merkle of small sequence (1000 digests)
+}
+
+}

libraries/chain/block_state.cpp

@@ -55,7 +55,7 @@ block_state::block_state(const block_header_state&                bhs,
 
 // Used for transition from dpos to Savanna.
 block_state_ptr block_state::create_if_genesis_block(const block_state_legacy& bsp) {
-   assert(bsp.action_receipt_digests_savanna);
+   assert(bsp.action_mroot_savanna);
 
    auto result_ptr = std::make_shared<block_state>();
    auto &result = *result_ptr;
@@ -86,15 +86,11 @@ block_state_ptr block_state::create_if_genesis_block(const block_state_legacy& b
    // TODO: Do not aggregate votes on blocks created from block_state_legacy. This can be removed when #2057 complete.
    result.pending_qc = pending_quorum_certificate{result.active_finalizer_policy->finalizers.size(), result.active_finalizer_policy->threshold, result.active_finalizer_policy->max_weak_sum_before_weak_final()};
 
-   // Calculate Merkle tree root in Savanna way so that it is stored in Leaf Node when building block_state.
-   const auto& digests = *bsp.action_receipt_digests_savanna;
-   auto action_mroot_svnn = calculate_merkle(digests);
-
    // build leaf_node and validation_tree
    valid_t::finality_leaf_node_t leaf_node {
       .block_num       = bsp.block_num(),
       .finality_digest = result.strong_digest,
-      .action_mroot    = action_mroot_svnn
+      .action_mroot    = *bsp.action_mroot_savanna
    };
    // construct valid structure
    incremental_merkle_tree validation_tree;
@@ -107,7 +103,7 @@ block_state_ptr block_state::create_if_genesis_block(const block_state_legacy& b
    result.validated.store(bsp.is_valid());
    result.pub_keys_recovered = bsp._pub_keys_recovered;
    result.cached_trxs = bsp._cached_trxs;
-   result.action_mroot = action_mroot_svnn;
+   result.action_mroot = *bsp.action_mroot_savanna;
    result.base_digest = {}; // calculated on demand in get_finality_data()
 
    return result_ptr;

libraries/chain/block_state_legacy.cpp

@@ -66,7 +66,7 @@ namespace eosio::chain {
    block_state_legacy::block_state_legacy( pending_block_header_state_legacy&& cur,
                                            signed_block_ptr&& b,
                                            deque<transaction_metadata_ptr>&& trx_metas,
-                                           std::optional<digests_t>&& action_receipt_digests_savanna,
+                                           const std::optional<digests_t>& action_receipt_digests_savanna,
                                            const protocol_feature_set& pfs,
                                            const validator_t& validator,
                                            const signer_callback_type& signer
@@ -75,7 +75,7 @@ namespace eosio::chain {
    ,block( std::move(b) )
    ,_pub_keys_recovered( true ) // called by produce_block so signature recovery of trxs must have been done
    ,_cached_trxs( std::move(trx_metas) )
-   ,action_receipt_digests_savanna( std::move(action_receipt_digests_savanna) )
+   ,action_mroot_savanna( action_receipt_digests_savanna ? std::optional<digest_type>(calculate_merkle(*action_receipt_digests_savanna)) : std::nullopt )
    {}
 
    block_state_legacy::block_state_legacy(snapshot_detail::snapshot_block_state_legacy_v7&& sbs)