IF: Move perf tests to benchmark

benchmark/benchmark.cpp

@@ -15,7 +15,8 @@ std::map<std::string, std::function<void()>> features {
    { "key", key_benchmarking },
    { "hash", hash_benchmarking },
    { "blake2", blake2_benchmarking },
-   { "bls", bls_benchmarking }
+   { "bls", bls_benchmarking },
+   { "merkle", merkle_benchmarking }
 };
 
 // values to control cout format
@@ -34,6 +35,10 @@ void set_num_runs(uint32_t runs) {
    num_runs = runs;
 }
 
+uint32_t get_num_runs() {
+   return num_runs;
+}
+
 void print_header() {
    std::cout << std::left << std::setw(name_width) << "function"
       << std::setw(runs_width) << "runs"
@@ -63,12 +68,14 @@ bytes to_bytes(const std::string& source) {
    return output;
 };
 
-void benchmarking(const std::string& name, const std::function<void()>& func) {
+void benchmarking(const std::string& name, const std::function<void()>& func,
+                  std::optional<size_t> opt_num_runs /* = {} */) {
    uint64_t total{0};
    uint64_t min{std::numeric_limits<uint64_t>::max()};
    uint64_t max{0};
+   uint32_t runs = opt_num_runs ? *opt_num_runs : num_runs;
 
-   for (auto i = 0U; i < num_runs; ++i) {
+   for (auto i = 0U; i < runs; ++i) {
       auto start_time = std::chrono::high_resolution_clock::now();
       func();
       auto end_time = std::chrono::high_resolution_clock::now();
@@ -79,7 +86,7 @@ void benchmarking(const std::string& name, const std::function<void()>& func) {
       max = std::max(max, duration);
    }
 
-   print_results(name, num_runs, total, min, max);
+   print_results(name, runs, total, min, max);
 }
 
 } // benchmark

benchmark/benchmark.hpp

@@ -4,13 +4,15 @@
 #include <map>
 #include <vector>
 #include <limits>
+#include <optional>
 
 #include <fc/crypto/hex.hpp>
 
 namespace eosio::benchmark {
 using bytes = std::vector<char>;
 
 void set_num_runs(uint32_t runs);
+uint32_t get_num_runs();
 std::map<std::string, std::function<void()>> get_features();
 void print_header();
 bytes to_bytes(const std::string& source);
@@ -21,7 +23,8 @@ void key_benchmarking();
 void hash_benchmarking();
 void blake2_benchmarking();
 void bls_benchmarking();
+void merkle_benchmarking();
 
-void benchmarking(const std::string& name, const std::function<void()>& func); 
+void benchmarking(const std::string& name, const std::function<void()>& func, std::optional<size_t> num_runs = {});
 
 } // benchmark

benchmark/merkle.cpp

@@ -0,0 +1,69 @@
+#include <benchmark.hpp>
+#include <eosio/chain/incremental_merkle.hpp>
+#include <eosio/chain/incremental_merkle_legacy.hpp>
+#include <random>
+
+namespace eosio::benchmark {
+
+using namespace eosio::chain;
+
+std::vector<digest_type> create_test_digests(size_t n) {
+   std::vector<digest_type> v;
+   v.reserve(n);
+   for (size_t i=0; i<n; ++i)
+      v.push_back(fc::sha256::hash(std::string{"Node"} + std::to_string(i)));
+   return v;
+}
+
+void benchmark_calc_merkle(uint32_t size_boost) {
+   using namespace std::string_literals;
+   const size_t num_digests = size_boost * 1000ull; // don't use exact powers of 2 as it is a special case
+
+   const std::vector<digest_type> digests = create_test_digests(num_digests);
+   const deque<digest_type> deq { digests.begin(), digests.end() };
+
+   auto num_str = std::to_string(size_boost);
+   while(num_str.size() < 4)
+      num_str.insert(0, 1, ' ');
+   auto msg_header = "Calc, "s + num_str + ",000 digests,  "s;
+   uint32_t num_runs = std::min(get_num_runs(), std::max(1u, get_num_runs() / size_boost));
+   benchmarking(msg_header + "legacy: ", [&]() { calculate_merkle_legacy(deq); }, num_runs);
+   benchmarking(msg_header + "savanna:", [&]() { calculate_merkle(digests.begin(), digests.end()); }, num_runs);
+}
+
+void benchmark_incr_merkle(uint32_t size_boost) {
+   using namespace std::string_literals;
+   const size_t num_digests = size_boost * 1000ull; // don't use exact powers of 2 as it is a special case
+
+   const std::vector<digest_type> digests = create_test_digests(num_digests);
+
+   auto num_str = std::to_string(size_boost);
+   while(num_str.size() < 4)
+      num_str.insert(0, 1, ' ');
+   auto msg_header = "Incr, "s + num_str + ",000 digests,  "s;
+   uint32_t num_runs = std::min(get_num_runs(), std::max(1u, get_num_runs() / size_boost));
+
+   auto incr = [&](const auto& incr_tree) {
+      auto work_tree = incr_tree;
+      for (const auto& d : digests)
+         work_tree.append(d);
+      return work_tree.get_root();
+   };
+
+   benchmarking(msg_header + "legacy: ", [&]() { incr(incremental_merkle_tree_legacy()); }, num_runs);
+   benchmarking(msg_header + "savanna:", [&]() { incr(incremental_merkle_tree()); }, num_runs);
+}
+
+// register benchmarking functions
+void merkle_benchmarking() {
+   benchmark_calc_merkle(1000); // calculate_merkle of very large sequence (1,000,000 digests)
+   benchmark_calc_merkle(50);   // calculate_merkle of large sequence (50,000 digests)
+   benchmark_calc_merkle(1);    // calculate_merkle of small sequence (1000 digests)
+   std::cout << "\n";
+
+   benchmark_incr_merkle(100);  // incremental_merkle of very large sequence (100,000 digests)
+   benchmark_incr_merkle(25);   // incremental_merkle of large sequence (25,000 digests)
+   benchmark_incr_merkle(1);    // incremental_merkle of small sequence (1000 digests)
+}
+
+}

unittests/merkle_tree_tests.cpp

@@ -2,7 +2,6 @@
 #include <eosio/chain/incremental_merkle_legacy.hpp>
 #include <boost/test/unit_test.hpp>
 #include <fc/crypto/sha256.hpp>
-#include <chrono>
 
 using namespace eosio::chain;
 using eosio::chain::detail::make_legacy_digest_pair;
@@ -181,112 +180,4 @@ BOOST_AUTO_TEST_CASE(consistency_over_large_range) {
    }
 }
 
-class stopwatch {
-public:
-   stopwatch(std::string msg) : _msg(std::move(msg)) {  _start = clock::now(); }
-
-   ~stopwatch() { std::cout << _msg << get_time_us()/1000000 << "s\n"; }
-
-   double get_time_us() const {
-      using duration_t = std::chrono::duration<double, std::micro>;
-      return std::chrono::duration_cast<duration_t>(clock::now() - _start).count();
-   }
-
-   using clock = std::chrono::high_resolution_clock;
-   using point = std::chrono::time_point<clock>;
-
-   std::string _msg;
-   point       _start;
-};
-
-BOOST_AUTO_TEST_CASE(perf_test_one_large) {
-   auto perf_test = [](const std::string& type, auto&& incr_tree, auto&& calc_fn) {
-      using namespace std::string_literals;
-      constexpr size_t num_digests = 1000ull * 1000ull; // don't use exact powers of 2 as it is a special case
-
-      const std::vector<digest_type> digests = create_test_digests(num_digests);
-      const deque<digest_type> deq { digests.begin(), digests.end() };
-
-      auto msg_header = "1 sequence of "s + std::to_string(num_digests) + " digests: time for "s;
-
-      auto incr_root = [&]() {
-         stopwatch s(msg_header + type + " incremental_merkle: ");
-         for (const auto& d : digests)
-            incr_tree.append(d);
-         return incr_tree.get_root();
-      }();
-
-      auto calc_root = [&]() {
-         stopwatch s(msg_header + type + " calculate_merkle:   ");
-         return calc_fn(deq);
-      }();
-
-      return std::make_pair(incr_root, calc_root);
-   };
-
-   {
-      auto [incr_root, calc_root] = perf_test("savanna", incremental_merkle_tree(),
-                                              [](const deque<digest_type>& d) { return calculate_merkle(d); }); // gcc10 needs a lambda here
-      BOOST_CHECK_EQUAL(incr_root, calc_root);
-   }
-
-   {
-      auto [incr_root, calc_root] = perf_test("legacy ", incremental_merkle_tree_legacy(), calculate_merkle_legacy);
-      BOOST_CHECK_EQUAL(incr_root, calc_root);
-   }
-}
-
-
-BOOST_AUTO_TEST_CASE(perf_test_many_small) {
-
-   auto perf_test = [](const std::string& type, const auto& incr_tree, auto&& calc_fn) {
-      using namespace std::string_literals;
-      constexpr size_t num_digests = 10000; // don't use exact powers of 2 as it is a special case
-      constexpr size_t num_runs    = 100;
-
-      const std::vector<digest_type> digests = create_test_digests(num_digests);
-      const deque<digest_type> deq { digests.begin(), digests.end() };
-
-      deque<digest_type> results(num_runs);
-
-      auto incr = [&]() {
-         auto work_tree = incr_tree;
-         for (const auto& d : digests)
-            work_tree.append(d);
-         return work_tree.get_root();
-      };
-
-      auto calc = [&]() { return calc_fn(deq); };
-
-      auto msg_header = std::to_string(num_runs) + " runs for a sequence of "s + std::to_string(num_digests) + " digests: time for "s;
-
-      auto incr_root = [&]() {
-         stopwatch s(msg_header + type + " incremental_merkle: ");
-         for (auto& r : results)
-            r = incr();
-         return calc_fn(results);
-      }();
-
-      auto calc_root = [&]() {
-         stopwatch s(msg_header + type + " calculate_merkle:   ");
-         for (auto& r : results)
-            r = calc();
-         return calc_fn(results);
-      }();
-
-      return std::make_pair(incr_root, calc_root);
-   };
-
-   {
-      auto [incr_root, calc_root] = perf_test("savanna", incremental_merkle_tree(),
-                                              [](const deque<digest_type>& d) { return calculate_merkle(d); }); // gcc10 needs a lambda here
-      BOOST_CHECK_EQUAL(incr_root, calc_root);
-   }
-
-   {
-      auto [incr_root, calc_root] = perf_test("legacy ", incremental_merkle_tree_legacy(), calculate_merkle_legacy);
-      BOOST_CHECK_EQUAL(incr_root, calc_root);
-   }
-}
-
 BOOST_AUTO_TEST_SUITE_END()