Merge pull request #2686 from bitshares/release

Merge release branch into testnet branch for test-6.1.1 pre-release for Testnet

.github/workflows/build-and-test.win.yml

@@ -7,7 +7,7 @@ env:
   BOOST_VERSION: 1_69_0
   BOOST_DOTTED_VERSION: 1.69.0
   CURL_VERSION: 7.86.0
-  OPENSSL_VERSION: 1.1.1q
+  OPENSSL_VERSION: 1.1.1s
   ZLIB_VERSION: 1.2.13
 jobs:
   prepare-mingw64-libs:

libraries/app/api.cpp

@@ -102,8 +102,8 @@ namespace graphene { namespace app {
 
     application_options login_api::get_config() const
     {
-       bool allowed = !_allowed_apis.empty();
-       FC_ASSERT( allowed, "Access denied, please login" );
+       bool is_allowed = !_allowed_apis.empty();
+       FC_ASSERT( is_allowed, "Access denied, please login" );
        return _app.get_options();
     }
 
@@ -114,8 +114,8 @@ namespace graphene { namespace app {
 
     bool login_api::is_database_api_allowed() const
     {
-       bool allowed = ( _allowed_apis.find("database_api") != _allowed_apis.end() );
-       return allowed;
+       bool is_allowed = ( _allowed_apis.find("database_api") != _allowed_apis.end() );
+       return is_allowed;
     }
 
     // block_api
@@ -244,8 +244,8 @@ namespace graphene { namespace app {
 
     fc::api<network_broadcast_api> login_api::network_broadcast()
     {
-       bool allowed = ( _allowed_apis.find("network_broadcast_api") != _allowed_apis.end() );
-       FC_ASSERT( allowed, "Access denied" );
+       bool is_allowed = ( _allowed_apis.find("network_broadcast_api") != _allowed_apis.end() );
+       FC_ASSERT( is_allowed, "Access denied" );
        if( !_network_broadcast_api )
        {
           _network_broadcast_api = std::make_shared< network_broadcast_api >( std::ref( _app ) );
@@ -255,8 +255,8 @@ namespace graphene { namespace app {
 
     fc::api<block_api> login_api::block()
     {
-       bool allowed = ( _allowed_apis.find("block_api") != _allowed_apis.end() );
-       FC_ASSERT( allowed, "Access denied" );
+       bool is_allowed = ( _allowed_apis.find("block_api") != _allowed_apis.end() );
+       FC_ASSERT( is_allowed, "Access denied" );
        if( !_block_api )
        {
           _block_api = std::make_shared< block_api >( std::ref( *_app.chain_database() ) );
@@ -266,8 +266,8 @@ namespace graphene { namespace app {
 
     fc::api<network_node_api> login_api::network_node()
     {
-       bool allowed = ( _allowed_apis.find("network_node_api") != _allowed_apis.end() );
-       FC_ASSERT( allowed, "Access denied" );
+       bool is_allowed = ( _allowed_apis.find("network_node_api") != _allowed_apis.end() );
+       FC_ASSERT( is_allowed, "Access denied" );
        if( !_network_node_api )
        {
           _network_node_api = std::make_shared< network_node_api >( std::ref(_app) );
@@ -277,8 +277,8 @@ namespace graphene { namespace app {
 
     fc::api<database_api> login_api::database()
     {
-       bool allowed = ( _allowed_apis.find("database_api") != _allowed_apis.end() );
-       FC_ASSERT( allowed, "Access denied" );
+       bool is_allowed = ( _allowed_apis.find("database_api") != _allowed_apis.end() );
+       FC_ASSERT( is_allowed, "Access denied" );
        if( !_database_api )
        {
           _database_api = std::make_shared< database_api >( std::ref( *_app.chain_database() ),
@@ -289,8 +289,8 @@ namespace graphene { namespace app {
 
     fc::api<history_api> login_api::history()
     {
-       bool allowed = ( _allowed_apis.find("history_api") != _allowed_apis.end() );
-       FC_ASSERT( allowed, "Access denied" );
+       bool is_allowed = ( _allowed_apis.find("history_api") != _allowed_apis.end() );
+       FC_ASSERT( is_allowed, "Access denied" );
        if( !_history_api )
        {
           _history_api = std::make_shared< history_api >( _app );
@@ -300,8 +300,8 @@ namespace graphene { namespace app {
 
     fc::api<crypto_api> login_api::crypto()
     {
-       bool allowed = ( _allowed_apis.find("crypto_api") != _allowed_apis.end() );
-       FC_ASSERT( allowed, "Access denied" );
+       bool is_allowed = ( _allowed_apis.find("crypto_api") != _allowed_apis.end() );
+       FC_ASSERT( is_allowed, "Access denied" );
        if( !_crypto_api )
        {
           _crypto_api = std::make_shared< crypto_api >();
@@ -311,8 +311,8 @@ namespace graphene { namespace app {
 
     fc::api<asset_api> login_api::asset()
     {
-       bool allowed = ( _allowed_apis.find("asset_api") != _allowed_apis.end() );
-       FC_ASSERT( allowed, "Access denied" );
+       bool is_allowed = ( _allowed_apis.find("asset_api") != _allowed_apis.end() );
+       FC_ASSERT( is_allowed, "Access denied" );
        if( !_asset_api )
        {
           _asset_api = std::make_shared< asset_api >( _app );
@@ -322,8 +322,8 @@ namespace graphene { namespace app {
 
     fc::api<orders_api> login_api::orders()
     {
-       bool allowed = ( _allowed_apis.find("orders_api") != _allowed_apis.end() );
-       FC_ASSERT( allowed, "Access denied" );
+       bool is_allowed = ( _allowed_apis.find("orders_api") != _allowed_apis.end() );
+       FC_ASSERT( is_allowed, "Access denied" );
        if( !_orders_api )
        {
           _orders_api = std::make_shared< orders_api >( std::ref( _app ) );
@@ -333,8 +333,8 @@ namespace graphene { namespace app {
 
     fc::api<graphene::debug_witness::debug_api> login_api::debug()
     {
-       bool allowed = ( _allowed_apis.find("debug_api") != _allowed_apis.end() );
-       FC_ASSERT( allowed, "Access denied" );
+       bool is_allowed = ( _allowed_apis.find("debug_api") != _allowed_apis.end() );
+       FC_ASSERT( is_allowed, "Access denied" );
        // can only use this API set if the plugin was loaded
        bool plugin_enabled = !!_app.get_plugin( "debug_witness" );
        FC_ASSERT( plugin_enabled, "The debug_witness plugin is not enabled" );
@@ -347,8 +347,8 @@ namespace graphene { namespace app {
 
     fc::api<custom_operations_api> login_api::custom_operations()
     {
-       bool allowed = ( _allowed_apis.find("custom_operations_api") != _allowed_apis.end() );
-       FC_ASSERT( allowed, "Access denied" );
+       bool is_allowed = ( _allowed_apis.find("custom_operations_api") != _allowed_apis.end() );
+       FC_ASSERT( is_allowed, "Access denied" );
        // can only use this API set if the plugin was loaded
        bool plugin_enabled = !!_app.get_plugin( "custom_operations" );
        FC_ASSERT( plugin_enabled, "The custom_operations plugin is not enabled" );

libraries/egenesis/seed-nodes.txt

@@ -1,4 +1,3 @@
-"seed01.liondani.com:1776",          // liondani       (Germany)
 "bts-seed1.abit-more.com:62015",     // abit           (Germany)
 "seed.roelandp.nl:1776",             // roelandp       (Canada)
 "seed1.xbts.io:1776",                // xbts.io        (Germany)

libraries/plugins/elasticsearch/elasticsearch_plugin.cpp

@@ -86,6 +86,7 @@ class elasticsearch_plugin_impl
       std::unique_ptr<graphene::utilities::es_client> es;
 
       vector <string> bulk_lines; //  vector of op lines
+      size_t approximate_bulk_size = 0;
 
       bulk_struct bulk_line_struct;
 
@@ -95,7 +96,7 @@ class elasticsearch_plugin_impl
 
       void add_elasticsearch( const account_id_type& account_id, const optional<operation_history_object>& oho,
                               uint32_t block_number );
-      void send_bulk();
+      void send_bulk( uint32_t block_num );
 
       void doOperationHistory(const optional <operation_history_object>& oho, operation_history_struct& os) const;
       void doBlock(uint32_t trx_in_block, const signed_block& b, block_struct& bs) const;
@@ -214,12 +215,14 @@ void elasticsearch_plugin_impl::update_account_histories( const signed_block& b
 
    // we send bulk at end of block when we are in sync for better real time client experience
    if( is_sync && !bulk_lines.empty() )
-      send_bulk();
+      send_bulk( b.block_num() );
 
 }
 
-void elasticsearch_plugin_impl::send_bulk()
+void elasticsearch_plugin_impl::send_bulk( uint32_t block_num )
 {
+   ilog( "Sending ${n} lines of bulk data to ElasticSearch at block ${b}, approximate size ${s}",
+         ("n",bulk_lines.size())("b",block_num)("s",approximate_bulk_size) );
    if( !es->send_bulk( bulk_lines ) )
    {
       elog( "Error sending ${n} lines of bulk data to ElasticSearch, the first lines are:",
@@ -233,6 +236,7 @@ void elasticsearch_plugin_impl::send_bulk()
             "Error populating ES database, we are going to keep trying." );
    }
    bulk_lines.clear();
+   approximate_bulk_size = 0;
    bulk_lines.reserve(limit_documents);
 }
 
@@ -435,8 +439,11 @@ void elasticsearch_plugin_impl::add_elasticsearch( const account_id_type& accoun
       auto prepare = graphene::utilities::createBulk(bulk_header, std::move(bulk_line));
       std::move(prepare.begin(), prepare.end(), std::back_inserter(bulk_lines));
 
-      if( bulk_lines.size() >= limit_documents )
-         send_bulk();
+      approximate_bulk_size += bulk_lines.back().size();
+
+      if( bulk_lines.size() >= limit_documents
+            || approximate_bulk_size >= graphene::utilities::es_client::request_size_threshold )
+         send_bulk( block_number );
    }
    cleanObjects(ath, account_id);
 }

libraries/plugins/es_objects/es_objects.cpp

@@ -128,8 +128,9 @@ class es_objects_plugin_impl
       std::unique_ptr<graphene::utilities::es_client> es;
 
       vector<std::string> bulk_lines;
+      size_t approximate_bulk_size = 0;
 
-      uint32_t block_number;
+      uint32_t block_number = 0;
       fc::time_point_sec block_time;
       bool is_es_version_7_or_above = true;
 
@@ -283,6 +284,8 @@ void es_objects_plugin_impl::delete_from_database(
 
    bulk_lines.push_back( fc::json::to_string(final_delete_line) );
 
+   approximate_bulk_size += bulk_lines.back().size();
+
    send_bulk_if_ready();
 }
 
@@ -324,14 +327,17 @@ void es_objects_plugin_impl::prepareTemplate(
    auto prepare = graphene::utilities::createBulk(bulk_header, std::move(data));
    std::move(prepare.begin(), prepare.end(), std::back_inserter(bulk_lines));
 
+   approximate_bulk_size += bulk_lines.back().size();
+
    send_bulk_if_ready();
 }
 
 void es_objects_plugin_impl::send_bulk_if_ready( bool force )
 {
    if( bulk_lines.empty() )
       return;
-   if( !force && bulk_lines.size() < limit_documents )
+   if( !force && bulk_lines.size() < limit_documents
+         && approximate_bulk_size < graphene::utilities::es_client::request_size_threshold )
       return;
    constexpr uint32_t log_count_threshold = 20000; // lines
    constexpr uint32_t log_time_threshold = 3600; // seconds
@@ -342,8 +348,10 @@ void es_objects_plugin_impl::send_bulk_if_ready( bool force )
    bool log_by_next = ( docs_sent_total >= next_log_count || fc::time_point::now() >= next_log_time );
    if( log_by_next || limit_documents == _options.bulk_replay || force )
    {
-      ilog( "Sending ${n} lines of bulk data to ElasticSearch, this batch ${b}, total ${t}",
-            ("n",bulk_lines.size())("b",docs_sent_batch)("t",docs_sent_total) );
+      ilog( "Sending ${n} lines of bulk data to ElasticSearch at block {blk}, "
+            "this batch ${b}, total ${t}, approximate size ${s}",
+            ("n",bulk_lines.size())("blk",block_number)
+            ("b",docs_sent_batch)("t",docs_sent_total)("s",approximate_bulk_size) );
       next_log_count = docs_sent_total + log_count_threshold;
       next_log_time = fc::time_point::now() + fc::seconds(log_time_threshold);
    }
@@ -362,6 +370,7 @@ void es_objects_plugin_impl::send_bulk_if_ready( bool force )
    }
    bulk_lines.clear();
    bulk_lines.reserve(limit_documents);
+   approximate_bulk_size = 0;
 }
 
 } // end namespace detail

libraries/utilities/elasticsearch.cpp

@@ -289,6 +289,7 @@ fc::variant es_data_adaptor::adapt( const fc::variant_object& op, uint16_t max_d
       { "active_special_authority", data_type::static_variant_type },
       { "owner_special_authority",  data_type::static_variant_type },
       { "htlc_preimage_hash",       data_type::static_variant_type },
+      { "argument",                 data_type::static_variant_type }, // for custom authority, restriction.argument
       { "feeds",                    data_type::map_type }, // asset_bitasset_data_object.feeds
       { "acceptable_collateral",    data_type::map_type },
       { "acceptable_borrowers",     data_type::map_type }
@@ -408,8 +409,9 @@ void es_data_adaptor::extract_data_from_variant(
       mv[prefix + "_string"] = v.get_string();
    else
       mv[prefix + "_string"] = fc::json::to_string( v );
-   // Note: we don't use double or array here, and we convert null and blob to string,
-   //       and static_variants (i.e. in custom authorities) and maps (if any) are converted to strings too.
+   // Note: we don't use double here, and we convert nulls and blobs to strings,
+   //       arrays and pairs (i.e. in custom authorities) are converted to strings,
+   //       static_variants and maps (if any) are converted to strings too.
 }
 
 fc::variant es_data_adaptor::adapt_map_item( const fc::variants& v, uint16_t max_depth )

libraries/utilities/include/graphene/utilities/elasticsearch.hpp

@@ -105,6 +105,9 @@ class es_client
    bool del( const std::string& path ) const;
    std::string get( const std::string& path ) const;
    std::string query( const std::string& path, const std::string& query ) const;
+
+   /// When doing bulk operations, call @ref send_bulk when the approximate size of pending data reaches this value.
+   static constexpr size_t request_size_threshold = 4 * 1024 * 1024; // 4MB
 private:
    std::string base_url;
    std::string auth;