From baddead071e579b614dd9804eb617e28ed5eefca Mon Sep 17 00:00:00 2001 From: battlmonstr Date: Mon, 30 Dec 2024 16:56:03 +0100 Subject: [PATCH] access_layer DataModel refactoring to use multi-bundle queries --- silkworm/db/access_layer.cpp | 49 +++++-------------- silkworm/db/blocks/headers/header_queries.hpp | 14 +++++- .../db/blocks/transactions/txn_queries.hpp | 12 +++-- .../db/datastore/snapshots/basic_queries.hpp | 12 ++--- silkworm/db/snapshot_test.cpp | 32 ++++++------ 5 files changed, 54 insertions(+), 65 deletions(-) diff --git a/silkworm/db/access_layer.cpp b/silkworm/db/access_layer.cpp index b8e31a691b..6f889c2992 100644 --- a/silkworm/db/access_layer.cpp +++ b/silkworm/db/access_layer.cpp @@ -1266,25 +1266,11 @@ bool DataModel::read_block_from_snapshot(BlockNum block_num, Block& block) const } std::optional DataModel::read_header_from_snapshot(BlockNum block_num) const { - std::optional block_header; - // We know the header snapshot in advance: find it based on target block number - const auto [segment_and_index, _] = repository_.find_segment(blocks::kHeaderSegmentAndIdxNames, block_num); - if (segment_and_index) { - block_header = HeaderFindByBlockNumSegmentQuery{*segment_and_index}.exec(block_num); - } - return block_header; + return HeaderFindByBlockNumQuery{repository_}.exec(block_num); } std::optional DataModel::read_header_from_snapshot(const Hash& hash) const { - std::optional block_header; - // We don't know the header snapshot in advance: search for block hash in each header snapshot in reverse order - for (const auto& bundle_ptr : repository_.view_bundles_reverse()) { - const auto& bundle = *bundle_ptr; - auto segment_and_index = bundle.segment_and_accessor_index(blocks::kHeaderSegmentAndIdxNames); - block_header = HeaderFindByHashSegmentQuery{segment_and_index}.exec(hash); - if (block_header) break; - } - return block_header; + return HeaderFindByHashQuery{repository_}.exec(hash); } std::optional DataModel::read_body_for_storage_from_snapshot(BlockNum block_num) const { @@ -1310,14 +1296,7 @@ bool DataModel::read_body_from_snapshot(BlockNum block_num, BlockBody& body) con } bool DataModel::is_body_in_snapshot(BlockNum block_num) const { - // We know the body snapshot in advance: find it based on target block number - const auto [segment_and_index, _] = repository_.find_segment(blocks::kBodySegmentAndIdxNames, block_num); - if (segment_and_index) { - const auto stored_body = BodyFindByBlockNumSegmentQuery{*segment_and_index}.exec(block_num); - return stored_body.has_value(); - } - - return false; + return BodyFindByBlockNumQuery{repository_}.exec(block_num).has_value(); } bool DataModel::read_transactions_from_snapshot(BlockNum block_num, uint64_t base_txn_id, uint64_t txn_count, std::vector& txs) const { @@ -1325,31 +1304,27 @@ bool DataModel::read_transactions_from_snapshot(BlockNum block_num, uint64_t bas return true; } - const auto [segment_and_index, _] = repository_.find_segment(blocks::kTxnSegmentAndIdxNames, block_num); - if (!segment_and_index) return false; - - txs = TransactionRangeFromIdSegmentQuery{*segment_and_index}.exec_into_vector(base_txn_id, txn_count); + auto txs_opt = TransactionRangeFromIdQuery{repository_}.exec(block_num, base_txn_id, txn_count); + if (!txs_opt) return false; + txs = std::move(*txs_opt); return true; } bool DataModel::read_rlp_transactions_from_snapshot(BlockNum block_num, std::vector& rlp_txs) const { - const auto [body_segment_and_index, _] = repository_.find_segment(blocks::kBodySegmentAndIdxNames, block_num); - if (body_segment_and_index) { - auto stored_body = BodyFindByBlockNumSegmentQuery{*body_segment_and_index}.exec(block_num); - if (!stored_body) return false; + auto stored_body = BodyFindByBlockNumQuery{repository_}.exec(block_num); + if (!stored_body) return false; + { // Skip first and last *system transactions* in block body const auto base_txn_id{stored_body->base_txn_id + 1}; const auto txn_count{stored_body->txn_count >= 2 ? stored_body->txn_count - 2 : stored_body->txn_count}; - if (txn_count == 0) return true; - const auto [tx_segment_and_index, _2] = repository_.find_segment(blocks::kTxnSegmentAndIdxNames, block_num); - if (!tx_segment_and_index) return false; - - rlp_txs = TransactionPayloadRlpRangeFromIdSegmentQuery{*tx_segment_and_index}.exec_into_vector(base_txn_id, txn_count); + auto txs_opt = TransactionPayloadRlpRangeFromIdQuery{repository_}.exec(block_num, base_txn_id, txn_count); + if (!txs_opt) return false; + rlp_txs = std::move(*txs_opt); return true; } diff --git a/silkworm/db/blocks/headers/header_queries.hpp b/silkworm/db/blocks/headers/header_queries.hpp index 03e1f66ad8..869b1cbea5 100644 --- a/silkworm/db/blocks/headers/header_queries.hpp +++ b/silkworm/db/blocks/headers/header_queries.hpp @@ -18,11 +18,21 @@ #include +#include "../schema_config.hpp" #include "header_segment.hpp" namespace silkworm::snapshots { -using HeaderFindByBlockNumSegmentQuery = FindByIdSegmentQuery; -using HeaderFindByHashSegmentQuery = FindByHashSegmentQuery; +using HeaderFindByBlockNumSegmentQuery = FindByIdSegmentQuery; + +struct HeaderFindByBlockNumQuery : public FindByTimestampMapQuery { + using FindByTimestampMapQuery::FindByTimestampMapQuery; + std::optional exec(BlockNum block_num) { + return FindByTimestampMapQuery::exec(block_num, block_num); + } +}; + +using HeaderFindByHashSegmentQuery = FindByHashSegmentQuery; +using HeaderFindByHashQuery = FindMapQuery; } // namespace silkworm::snapshots diff --git a/silkworm/db/blocks/transactions/txn_queries.hpp b/silkworm/db/blocks/transactions/txn_queries.hpp index 4717450575..56bee93081 100644 --- a/silkworm/db/blocks/transactions/txn_queries.hpp +++ b/silkworm/db/blocks/transactions/txn_queries.hpp @@ -28,10 +28,14 @@ namespace silkworm::snapshots { -using TransactionFindByIdSegmentQuery = FindByIdSegmentQuery; -using TransactionFindByHashSegmentQuery = FindByHashSegmentQuery; -using TransactionRangeFromIdSegmentQuery = RangeFromIdSegmentQuery; -using TransactionPayloadRlpRangeFromIdSegmentQuery = RangeFromIdSegmentQuery>; +using TransactionFindByIdSegmentQuery = FindByIdSegmentQuery; +using TransactionFindByHashSegmentQuery = FindByHashSegmentQuery; + +using TransactionRangeFromIdSegmentQuery = RangeFromIdSegmentQuery; +using TransactionRangeFromIdQuery = FindByTimestampMapQuery; + +using TransactionPayloadRlpRangeFromIdSegmentQuery = RangeFromIdSegmentQuery, &db::blocks::kTxnSegmentAndIdxNames>; +using TransactionPayloadRlpRangeFromIdQuery = FindByTimestampMapQuery; class TransactionBlockNumByTxnHashSegmentQuery { public: diff --git a/silkworm/db/datastore/snapshots/basic_queries.hpp b/silkworm/db/datastore/snapshots/basic_queries.hpp index d3288f1e08..6f85179e7c 100644 --- a/silkworm/db/datastore/snapshots/basic_queries.hpp +++ b/silkworm/db/datastore/snapshots/basic_queries.hpp @@ -30,7 +30,7 @@ namespace silkworm::snapshots { template < segment::SegmentReaderConcept TSegmentReader, - const SegmentAndAccessorIndexNames* segment_names = nullptr> + const SegmentAndAccessorIndexNames* segment_names> class BasicSegmentQuery { public: explicit BasicSegmentQuery( @@ -48,7 +48,7 @@ class BasicSegmentQuery { template < segment::SegmentReaderConcept TSegmentReader, - const SegmentAndAccessorIndexNames* segment_names = nullptr> + const SegmentAndAccessorIndexNames* segment_names> struct FindByIdSegmentQuery : public BasicSegmentQuery { using BasicSegmentQuery::BasicSegmentQuery; @@ -64,7 +64,7 @@ struct FindByIdSegmentQuery : public BasicSegmentQuery + const SegmentAndAccessorIndexNames* segment_names> struct FindByHashSegmentQuery : public BasicSegmentQuery { using BasicSegmentQuery::BasicSegmentQuery; @@ -87,14 +87,14 @@ struct FindByHashSegmentQuery : public BasicSegmentQuery + const SegmentAndAccessorIndexNames* segment_names> struct RangeFromIdSegmentQuery : public BasicSegmentQuery { using BasicSegmentQuery::BasicSegmentQuery; - std::vector exec_into_vector(uint64_t first_id, uint64_t count) { + std::optional> exec(uint64_t first_id, uint64_t count) { auto offset = this->index_.lookup_by_data_id(first_id); if (!offset) { - return {}; + return std::nullopt; } return this->reader_.read_into_vector(*offset, count); diff --git a/silkworm/db/snapshot_test.cpp b/silkworm/db/snapshot_test.cpp index 357f19d0db..91c3c20674 100644 --- a/silkworm/db/snapshot_test.cpp +++ b/silkworm/db/snapshot_test.cpp @@ -211,18 +211,18 @@ TEST_CASE("TransactionSnapshot::txn_range OK", "[silkworm][node][snapshot][index TransactionRangeFromIdSegmentQuery query{{txn_segment, idx_txn_hash}}; // block 1'500'012: base_txn_id is 7'341'263, txn_count is 7 - CHECK(query.exec_into_vector(7'341'263, 0).empty()); - CHECK(query.exec_into_vector(7'341'263, 7).size() == 7); + CHECK(query.exec(7'341'263, 0)->empty()); + CHECK(query.exec(7'341'263, 7)->size() == 7); // block 1'500'013: base_txn_id is 7'341'272, txn_count is 1 - CHECK(query.exec_into_vector(7'341'272, 0).empty()); - CHECK(query.exec_into_vector(7'341'272, 1).size() == 1); + CHECK(query.exec(7'341'272, 0)->empty()); + CHECK(query.exec(7'341'272, 1)->size() == 1); // invalid base_txn_id returns empty - CHECK(query.exec_into_vector(0, 1).empty()); - CHECK(query.exec_into_vector(10'000'000, 1).empty()); - CHECK(query.exec_into_vector(7'341'261, 1).empty()); // before the first system tx - CHECK(query.exec_into_vector(7'341'274, 1).empty()); // after the last system tx + CHECK_FALSE(query.exec(0, 1)); + CHECK_FALSE(query.exec(10'000'000, 1)); + CHECK_FALSE(query.exec(7'341'261, 1)); // before the first system tx + CHECK_FALSE(query.exec(7'341'274, 1)); // after the last system tx } TEST_CASE("TransactionSnapshot::txn_rlp_range OK", "[silkworm][node][snapshot][index]") { @@ -240,18 +240,18 @@ TEST_CASE("TransactionSnapshot::txn_rlp_range OK", "[silkworm][node][snapshot][i TransactionPayloadRlpRangeFromIdSegmentQuery query{{txn_segment, idx_txn_hash}}; // block 1'500'012: base_txn_id is 7'341'263, txn_count is 7 - CHECK(query.exec_into_vector(7'341'263, 0).empty()); - CHECK(query.exec_into_vector(7'341'263, 7).size() == 7); + CHECK(query.exec(7'341'263, 0)->empty()); + CHECK(query.exec(7'341'263, 7)->size() == 7); // block 1'500'013: base_txn_id is 7'341'272, txn_count is 1 - CHECK(query.exec_into_vector(7'341'272, 0).empty()); - CHECK(query.exec_into_vector(7'341'272, 1).size() == 1); + CHECK(query.exec(7'341'272, 0)->empty()); + CHECK(query.exec(7'341'272, 1)->size() == 1); // invalid base_txn_id returns empty - CHECK(query.exec_into_vector(0, 1).empty()); - CHECK(query.exec_into_vector(10'000'000, 1).empty()); - CHECK(query.exec_into_vector(7'341'261, 1).empty()); // before the first system tx - CHECK(query.exec_into_vector(7'341'274, 1).empty()); // after the last system tx + CHECK_FALSE(query.exec(0, 1)); + CHECK_FALSE(query.exec(10'000'000, 1)); + CHECK_FALSE(query.exec(7'341'261, 1)); // before the first system tx + CHECK_FALSE(query.exec(7'341'274, 1)); // after the last system tx } TEST_CASE("slice_tx_payload", "[silkworm][node][snapshot]") {