From 29d3859519e199ed57d2e84acecf52c2f5852819 Mon Sep 17 00:00:00 2001 From: Lee *!* Clagett Date: Tue, 5 Dec 2023 20:23:50 -0500 Subject: [PATCH] Add (working draft) subaddress support (#83) --- src/db/account.cpp | 21 +- src/db/account.h | 12 +- src/db/data.cpp | 108 +++++++- src/db/data.h | 64 ++++- src/db/fwd.h | 12 + src/db/storage.cpp | 329 ++++++++++++++++++++++- src/db/storage.h | 27 ++ src/error.cpp | 4 + src/error.h | 2 + src/lmdb/msgpack_table.h | 16 +- src/rest_server.cpp | 149 ++++++++++- src/rest_server.h | 1 + src/rpc/light_wallet.cpp | 44 +++- src/rpc/light_wallet.h | 40 +++ src/scanner.cpp | 191 ++++++++++---- src/scanner.h | 2 +- src/server_main.cpp | 9 +- src/wire/adapted/array.h | 96 +++++++ tests/unit/db/CMakeLists.txt | 2 +- tests/unit/db/data.test.cpp | 76 ++++++ tests/unit/db/subaddress.test.cpp | 422 ++++++++++++++++++++++++++++++ 21 files changed, 1539 insertions(+), 88 deletions(-) create mode 100644 src/wire/adapted/array.h create mode 100644 tests/unit/db/data.test.cpp create mode 100644 tests/unit/db/subaddress.test.cpp diff --git a/src/db/account.cpp b/src/db/account.cpp index 2c99894..79c6bba 100644 --- a/src/db/account.cpp +++ b/src/db/account.cpp @@ -72,11 +72,11 @@ namespace lws crypto::secret_key view_key; }; - account::account(std::shared_ptr immutable, db::block_id height, std::vector spendable, std::vector pubs) noexcept + account::account(std::shared_ptr immutable, db::block_id height, std::vector> spendable, std::vector pubs) noexcept : immutable_(std::move(immutable)) , spendable_(std::move(spendable)) , pubs_(std::move(pubs)) - , spends_() + , spends_() , outputs_() , height_(height) {} @@ -87,7 +87,7 @@ namespace lws MONERO_THROW(::common_error::kInvalidArgument, "using moved from account"); } - account::account(db::account const& source, std::vector spendable, std::vector pubs) + account::account(db::account const& source, std::vector> spendable, std::vector pubs) : account(std::make_shared(source), source.scan_height, std::move(spendable), std::move(pubs)) { std::sort(spendable_.begin(), spendable_.end()); @@ -151,9 +151,15 @@ namespace lws return immutable_->view_key; } - bool account::has_spendable(db::output_id const& id) const noexcept + boost::optional account::get_spendable(db::output_id const& id) const noexcept { - return std::binary_search(spendable_.begin(), spendable_.end(), id); + const auto searchable = + std::make_pair(id, db::address_index{db::major_index::primary, db::minor_index::primary}); + const auto account = + std::lower_bound(spendable_.begin(), spendable_.end(), searchable); + if (account == spendable_.end() || account->first != id) + return boost::none; + return account->second; } bool account::add_out(db::output const& out) @@ -163,9 +169,10 @@ namespace lws return false; pubs_.insert(existing_pub, out.pub); + auto spendable_value = std::make_pair(out.spend_meta.id, out.recipient); spendable_.insert( - std::lower_bound(spendable_.begin(), spendable_.end(), out.spend_meta.id), - out.spend_meta.id + std::lower_bound(spendable_.begin(), spendable_.end(), spendable_value), + spendable_value ); outputs_.push_back(out); return true; diff --git a/src/db/account.h b/src/db/account.h index 9a18d95..8e630ea 100644 --- a/src/db/account.h +++ b/src/db/account.h @@ -26,6 +26,7 @@ // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once +#include #include #include #include @@ -33,6 +34,7 @@ #include "crypto/crypto.h" #include "fwd.h" +#include "db/data.h" #include "db/fwd.h" namespace lws @@ -43,19 +45,19 @@ namespace lws struct internal; std::shared_ptr immutable_; - std::vector spendable_; + std::vector> spendable_; std::vector pubs_; std::vector spends_; std::vector outputs_; db::block_id height_; - explicit account(std::shared_ptr immutable, db::block_id height, std::vector spendable, std::vector pubs) noexcept; + explicit account(std::shared_ptr immutable, db::block_id height, std::vector> spendable, std::vector pubs) noexcept; void null_check() const; public: //! Construct an account from `source` and current `spendable` outputs. - explicit account(db::account const& source, std::vector spendable, std::vector pubs); + explicit account(db::account const& source, std::vector> spendable, std::vector pubs); /*! \return False if this is a "moved-from" account (i.e. the internal memory @@ -96,8 +98,8 @@ namespace lws //! \return Current scan height of `this`. db::block_id scan_height() const noexcept { return height_; } - //! \return True iff `id` is spendable by `this`. - bool has_spendable(db::output_id const& id) const noexcept; + //! \return Subaddress index iff `id` is spendable by `this`. + boost::optional get_spendable(db::output_id const& id) const noexcept; //! \return Outputs matched during the latest scan. std::vector const& outputs() const noexcept { return outputs_; } diff --git a/src/db/data.cpp b/src/db/data.cpp index 1aa730f..cba0456 100644 --- a/src/db/data.cpp +++ b/src/db/data.cpp @@ -29,12 +29,18 @@ #include #include +#include "cryptonote_config.h" // monero/src #include "db/string.h" +#include "int-util.h" // monero/contribe/epee/include +#include "ringct/rctOps.h" // monero/src +#include "ringct/rctTypes.h" // monero/src #include "wire.h" +#include "wire/adapted/array.h" #include "wire/crypto.h" #include "wire/json/write.h" #include "wire/msgpack.h" #include "wire/uuid.h" +#include "wire/vector.h" #include "wire/wrapper/defaulted.h" namespace lws @@ -69,6 +75,102 @@ namespace db } WIRE_DEFINE_OBJECT(account_address, map_account_address); + namespace + { + template + void map_subaddress_dict(F& format, T& self) + { + wire::object(format, + wire::field<0>("key", std::ref(self.first)), + wire::field<1>("value", std::ref(self.second)) + ); + } + } + + bool check_subaddress_dict(const subaddress_dict& self) + { + bool is_first = true; + minor_index last = minor_index::primary; + for (const auto& elem : self.second) + { + if (elem[1] < elem[0]) + { + MERROR("Invalid subaddress_range (last before first"); + return false; + } + if (std::uint32_t(elem[0]) <= std::uint64_t(last) + 1 && !is_first) + { + MERROR("Invalid subaddress_range (overlapping with previous)"); + return false; + } + is_first = false; + last = elem[1]; + } + return true; + } + void read_bytes(wire::reader& source, subaddress_dict& dest) + { + map_subaddress_dict(source, dest); + if (!check_subaddress_dict(dest)) + WIRE_DLOG_THROW_(wire::error::schema::array); + } + void write_bytes(wire::writer& dest, const subaddress_dict& source) + { + if (!check_subaddress_dict(source)) + WIRE_DLOG_THROW_(wire::error::schema::array); + map_subaddress_dict(dest, source); + } + + namespace + { + template + void map_address_index(F& format, T& self) + { + wire::object(format, WIRE_FIELD_ID(0, maj_i), WIRE_FIELD_ID(1, min_i)); + } + + crypto::secret_key get_subaddress_secret_key(const crypto::secret_key &a, const std::uint32_t major, const std::uint32_t minor) + { + char data[sizeof(config::HASH_KEY_SUBADDRESS) + sizeof(crypto::secret_key) + 2 * sizeof(uint32_t)]; + memcpy(data, config::HASH_KEY_SUBADDRESS, sizeof(config::HASH_KEY_SUBADDRESS)); + memcpy(data + sizeof(config::HASH_KEY_SUBADDRESS), &a, sizeof(crypto::secret_key)); + std::uint32_t idx = SWAP32LE(major); + memcpy(data + sizeof(config::HASH_KEY_SUBADDRESS) + sizeof(crypto::secret_key), &idx, sizeof(uint32_t)); + idx = SWAP32LE(minor); + memcpy(data + sizeof(config::HASH_KEY_SUBADDRESS) + sizeof(crypto::secret_key) + sizeof(uint32_t), &idx, sizeof(uint32_t)); + crypto::secret_key m; + crypto::hash_to_scalar(data, sizeof(data), m); + return m; + } + } + WIRE_DEFINE_OBJECT(address_index, map_address_index); + + crypto::public_key address_index::get_spend_public(account_address const& base, crypto::secret_key const& view) const + { + if (is_zero()) + return base.spend_public; + + // m = Hs(a || index_major || index_minor) + crypto::secret_key m = get_subaddress_secret_key(view, std::uint32_t(maj_i), std::uint32_t(min_i)); + + // M = m*G + crypto::public_key M; + crypto::secret_key_to_public_key(m, M); + + // D = B + M + return rct::rct2pk(rct::addKeys(rct::pk2rct(base.spend_public), rct::pk2rct(M))); + } + + namespace + { + template + void map_subaddress_map(F& format, T& self) + { + wire::object(format, WIRE_FIELD_ID(0, subaddress), WIRE_FIELD_ID(1, index)); + } + } + WIRE_DEFINE_OBJECT(subaddress_map, map_subaddress_map); + void write_bytes(wire::writer& dest, const account& self, const bool show_key) { view_key const* const key = @@ -144,7 +246,8 @@ namespace db wire::field<10>("unlock_time", self.unlock_time), wire::field<11>("mixin_count", self.spend_meta.mixin_count), wire::field<12>("coinbase", coinbase), - wire::field<13>("fee", self.fee) + wire::field<13>("fee", self.fee), + wire::field<14>("recipient", self.recipient) ); } @@ -161,7 +264,8 @@ namespace db WIRE_FIELD(timestamp), WIRE_FIELD(unlock_time), WIRE_FIELD(mixin_count), - wire::optional_field("payment_id", std::ref(payment_id)) + wire::optional_field("payment_id", std::ref(payment_id)), + WIRE_FIELD(sender) ); } } diff --git a/src/db/data.h b/src/db/data.h index ef741cd..0a55068 100644 --- a/src/db/data.h +++ b/src/db/data.h @@ -26,13 +26,16 @@ // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once +#include #include #include #include #include #include #include +#include #include +#include #include "crypto/crypto.h" #include "lmdb/util.h" @@ -122,6 +125,49 @@ namespace db static_assert(sizeof(account_address) == 64, "padding in account_address"); WIRE_DECLARE_OBJECT(account_address); + //! Major index of a subaddress + enum class major_index : std::uint32_t { primary = 0 }; + WIRE_AS_INTEGER(major_index); + + //! Minor index of a subaddress + enum class minor_index : std::uint32_t { primary = 0 }; + WIRE_AS_INTEGER(minor_index); + + //! Range within a major index + using index_range = std::array; + + //! Ranges within a major index + using index_ranges = std::vector; + + //! Compatible with msgpack_table + using subaddress_dict = std::pair; + bool check_subaddress_dict(const subaddress_dict&); + WIRE_DECLARE_OBJECT(subaddress_dict); + + //! A specific (sub)address index + struct address_index + { + major_index maj_i; + minor_index min_i; + + crypto::public_key get_spend_public(account_address const& base, crypto::secret_key const& view) const; + constexpr bool is_zero() const noexcept + { + return maj_i == major_index::primary && min_i == minor_index::primary; + } + }; + static_assert(sizeof(address_index) == 4 * 2, "padding in address_index"); + WIRE_DECLARE_OBJECT(address_index); + + //! Maps a subaddress pubkey to its index values + struct subaddress_map + { + crypto::public_key subaddress; //!< Must be first for LMDB optimzations + address_index index; + }; + static_assert(sizeof(subaddress_map) == 32 + 4 * 2, "padding in subaddress_map"); + WIRE_DECLARE_OBJECT(subaddress_map); + struct account { account_id id; //!< Must be first for LMDB optimizations @@ -205,9 +251,10 @@ namespace db crypto::hash long_; //!< Long version of payment id (always decrypted) } payment_id; std::uint64_t fee; //!< Total fee for transaction + address_index recipient; }; static_assert( - sizeof(output) == 8 + 32 + (8 * 3) + (4 * 2) + 32 + (8 * 2) + (32 * 3) + 7 + 1 + 32 + 8, + sizeof(output) == 8 + 32 + (8 * 3) + (4 * 2) + 32 + (8 * 2) + (32 * 3) + 7 + 1 + 32 + 8 + 2 * 4, "padding in output" ); void write_bytes(wire::writer&, const output&); @@ -225,8 +272,9 @@ namespace db char reserved[3]; std::uint8_t length; //!< Length of `payment_id` field (0..32). crypto::hash payment_id; //!< Unencrypted only, can't decrypt spend + address_index sender; }; - static_assert(sizeof(spend) == 8 + 32 * 2 + 8 * 4 + 4 + 3 + 1 + 32, "padding in spend"); + static_assert(sizeof(spend) == 8 + 32 * 2 + 8 * 4 + 4 + 3 + 1 + 32 + 2 * 4, "padding in spend"); WIRE_DECLARE_OBJECT(spend); //! Key image and info needed to retrieve primary `spend` data. @@ -325,6 +373,18 @@ namespace db }; void write_bytes(wire::writer&, const webhook_new_account&); + inline constexpr bool operator==(address_index const& left, address_index const& right) noexcept + { + return left.maj_i == right.maj_i && left.min_i == right.min_i; + } + + inline constexpr bool operator<(address_index const& left, address_index const& right) noexcept + { + return left.maj_i == right.maj_i ? + left.min_i < right.min_i : left.maj_i < right.maj_i; + } + + bool operator==(transaction_link const& left, transaction_link const& right) noexcept; bool operator<(transaction_link const& left, transaction_link const& right) noexcept; bool operator<=(transaction_link const& left, transaction_link const& right) noexcept; diff --git a/src/db/fwd.h b/src/db/fwd.h index f5eaa93..3027823 100644 --- a/src/db/fwd.h +++ b/src/db/fwd.h @@ -39,10 +39,14 @@ namespace db enum class block_id : std::uint64_t; enum extra : std::uint8_t; enum class extra_and_length : std::uint8_t; + enum class major_index : std::uint32_t; + enum class minor_index : std::uint32_t; enum class request : std::uint8_t; + enum class webhook_type : std::uint8_t; struct account; struct account_address; + struct address_index; struct block_info; struct key_image; struct output; @@ -50,7 +54,15 @@ namespace db struct request_info; struct spend; class storage; + struct subaddress_map; struct transaction_link; struct view_key; + struct webhook_data; + struct webhook_dupsort; + struct webhook_event; + struct webhook_key; + struct webhook_new_account; + struct webhook_output; + struct webhook_tx_confirmation; } // db } // lws diff --git a/src/db/storage.cpp b/src/db/storage.cpp index d537840..0a84560 100644 --- a/src/db/storage.cpp +++ b/src/db/storage.cpp @@ -57,6 +57,7 @@ #include "lmdb/value_stream.h" #include "net/net_parse_helpers.h" // monero/contrib/epee/include #include "span.h" +#include "wire/adapted/array.h" #include "wire/filters.h" #include "wire/json.h" #include "wire/vector.h" @@ -102,8 +103,63 @@ namespace db sizeof(output) == 8 + 32 + (8 * 3) + (4 * 2) + 32 + (8 * 2) + (32 * 3) + 7 + 1 + 32, "padding in output" ); + + //! Original db value, with no subaddress + struct spend + { + transaction_link link; //!< Orders and links `spend` to `output`. + crypto::key_image image; //!< Unique ID for the spend + // `link` and `image` must in this order for LMDB optimizations + output_id source; //!< The output being spent + std::uint64_t timestamp; //!< Timestamp of spend + std::uint64_t unlock_time;//!< Unlock time of spend + std::uint32_t mixin_count;//!< Ring-size of TX output + char reserved[3]; + std::uint8_t length; //!< Length of `payment_id` field (0..32). + crypto::hash payment_id; //!< Unencrypted only, can't decrypt spend + }; + static_assert(sizeof(spend) == 8 + 32 * 2 + 8 * 4 + 4 + 3 + 1 + 32, "padding in spend"); } + namespace v1 + { + //! Second DB value, with no subaddress + struct output + { + transaction_link link; //! Orders and links `output` to `spend`s. + + //! Data that a linked `spend` needs in some REST endpoints. + struct spend_meta_ + { + output_id id; //!< Unique id for output within monero + // `link` and `id` must be in this order for LMDB optimizations + std::uint64_t amount; + std::uint32_t mixin_count;//!< Ring-size of TX + std::uint32_t index; //!< Offset within a tx + crypto::public_key tx_public; + } spend_meta; + + std::uint64_t timestamp; + std::uint64_t unlock_time; //!< Not always a timestamp; mirrors chain value. + crypto::hash tx_prefix_hash; + crypto::public_key pub; //!< One-time spendable public key. + rct::key ringct_mask; //!< Unencrypted CT mask + char reserved[7]; + extra_and_length extra; //!< Extra info + length of payment id + union payment_id_ + { + crypto::hash8 short_; //!< Decrypted short payment id + crypto::hash long_; //!< Long version of payment id (always decrypted) + } payment_id; + std::uint64_t fee; //!< Total fee for transaction + }; + static_assert( + sizeof(output) == 8 + 32 + (8 * 3) + (4 * 2) + 32 + (8 * 2) + (32 * 3) + 7 + 1 + 32 + 8, + "padding in output" + ); + } + + namespace { //! Used for finding `account` instances by other indexes. @@ -243,11 +299,17 @@ namespace db constexpr const lmdb::basic_table outputs_v0{ "outputs_by_account_id,block_id,tx_hash,output_id", MDB_DUPSORT, &output_compare }; + constexpr const lmdb::basic_table outputs_v1{ + "outputs_v1_by_account_id,block_id,tx_hash,output_id", MDB_DUPSORT, &output_compare + }; constexpr const lmdb::basic_table outputs{ - "outputs_v1_by_account_id,block_id,tx_hash,output_id", (MDB_CREATE | MDB_DUPSORT), &output_compare + "outputs_v2_by_account_id,block_id,tx_hash,output_id", (MDB_CREATE | MDB_DUPSORT), &output_compare + }; + constexpr const lmdb::basic_table spends_v0{ + "spends_by_account_id,block_id,tx_hash,image", MDB_DUPSORT, &spend_compare }; constexpr const lmdb::basic_table spends{ - "spends_by_account_id,block_id,tx_hash,image", (MDB_CREATE | MDB_DUPSORT), &spend_compare + "spends_v1_by_account_id,block_id,tx_hash,image", (MDB_CREATE | MDB_DUPSORT), &spend_compare }; constexpr const lmdb::basic_table images{ "key_images_by_output_id,image", (MDB_CREATE | MDB_DUPSORT), MONERO_COMPARE(db::key_image, value) @@ -261,6 +323,12 @@ namespace db constexpr const lmdb::basic_table events_by_account_id{ "webhook_events_by_account_id,type,block_id,tx_hash,output_id,payment_id,event_id", (MDB_CREATE | MDB_DUPSORT), &lmdb::less }; + constexpr const lmdb::msgpack_table subaddress_ranges{ + "subaddress_ranges_by_account_id,major_index", (MDB_CREATE | MDB_DUPSORT), &lmdb::less + }; + constexpr const lmdb::basic_table subaddress_indexes{ + "subaddress_indexes_by_account_id,public_key", (MDB_CREATE | MDB_DUPSORT), MONERO_COMPARE(subaddress_map, subaddress) + }; template expect check_cursor(MDB_txn& txn, MDB_dbi tbl, std::unique_ptr& cur) noexcept @@ -553,6 +621,8 @@ namespace db MDB_dbi requests; MDB_dbi webhooks; MDB_dbi events; + MDB_dbi subaddress_ranges; + MDB_dbi subaddress_indexes; } tables; const unsigned create_queue_max; @@ -573,6 +643,8 @@ namespace db tables.requests = requests.open(*txn).value(); tables.webhooks = webhooks.open(*txn).value(); tables.events = events_by_account_id.open(*txn).value(); + tables.subaddress_ranges = subaddress_ranges.open(*txn).value(); + tables.subaddress_indexes = subaddress_indexes.open(*txn).value(); const auto v0_outputs = outputs_v0.open(*txn); if (v0_outputs) @@ -580,6 +652,18 @@ namespace db else if (v0_outputs != lmdb::error(MDB_NOTFOUND)) MONERO_THROW(v0_outputs.error(), "Error opening old outputs table"); + const auto v1_outputs = outputs_v1.open(*txn); + if (v1_outputs) + MONERO_UNWRAP(convert_table(*txn, *v1_outputs, tables.outputs)); + else if (v1_outputs != lmdb::error(MDB_NOTFOUND)) + MONERO_THROW(v1_outputs.error(), "Error opening old outputs table"); + + const auto v0_spends = spends_v0.open(*txn); + if (v0_spends) + MONERO_UNWRAP(convert_table(*txn, *v0_spends, tables.spends)); + else if (v0_spends != lmdb::error(MDB_NOTFOUND)) + MONERO_THROW(v0_spends.error(), "Error opening old spends table"); + check_blockchain(*txn, tables.blocks); MONERO_UNWRAP(this->commit(std::move(txn))); @@ -755,6 +839,52 @@ namespace db return requests.get_value(value); } + expect> + storage_reader::get_subaddresses(account_id id, cursor::subaddress_ranges cur) noexcept + { + MONERO_PRECOND(txn != nullptr); + assert(db != nullptr); + + MONERO_CHECK(check_cursor(*txn, db->tables.subaddress_ranges, cur)); + + MDB_val key = lmdb::to_val(id); + MDB_val value{}; + std::vector ranges{}; + int err = mdb_cursor_get(cur.get(), &key, &value, MDB_SET_KEY); + if (!err) + { + std::size_t count = 0; + if (mdb_cursor_count(cur.get(), &count) == 0) + ranges.reserve(count); + } + for (;;) + { + if (err) + { + if (err == MDB_NOTFOUND) + break; + return {lmdb::error(err)}; + } + ranges.push_back(MONERO_UNWRAP(subaddress_ranges.get_value(value))); + err = mdb_cursor_get(cur.get(), &key, &value, MDB_NEXT_DUP); + } + return {std::move(ranges)}; + } + + expect + storage_reader::find_subaddress(account_id id, crypto::public_key const& address, cursor::subaddress_indexes& cur) noexcept + { + MONERO_PRECOND(txn != nullptr); + assert(db != nullptr); + + MONERO_CHECK(check_cursor(*txn, db->tables.subaddress_indexes, cur)); + MDB_val key = lmdb::to_val(id); + MDB_val value = lmdb::to_val(address); + + MONERO_LMDB_CHECK(mdb_cursor_get(cur.get(), &key, &value, MDB_GET_BOTH)); + return subaddress_indexes.get_value(value); + } + expect> storage_reader::find_webhook(webhook_key const& key, crypto::hash8 const& payment_id, cursor::webhooks cur) { @@ -889,6 +1019,14 @@ namespace db ); } + static void write_bytes(wire::json_writer& dest, const std::pair>>>>& self) + { + wire::object(dest, + wire::field("id", std::cref(self.first)), + wire::field("subaddress_indexes", std::cref(self.second)) + ); + } + expect storage_reader::json_debug(std::ostream& out, bool show_keys) { using boost::adaptors::reverse; @@ -909,6 +1047,8 @@ namespace db cursor::requests requests_cur; cursor::webhooks webhooks_cur; cursor::webhooks events_cur; + cursor::subaddress_ranges ranges_cur; + cursor::subaddress_indexes indexes_cur; MONERO_CHECK(check_cursor(*txn, db->tables.blocks, curs.blocks_cur)); MONERO_CHECK(check_cursor(*txn, db->tables.accounts, accounts_cur)); @@ -920,6 +1060,8 @@ namespace db MONERO_CHECK(check_cursor(*txn, db->tables.requests, requests_cur)); MONERO_CHECK(check_cursor(*txn, db->tables.webhooks, webhooks_cur)); MONERO_CHECK(check_cursor(*txn, db->tables.events, events_cur)); + MONERO_CHECK(check_cursor(*txn, db->tables.subaddress_ranges, ranges_cur)); + MONERO_CHECK(check_cursor(*txn, db->tables.subaddress_indexes, indexes_cur)); auto blocks_partial = get_blocks>(*curs.blocks_cur, 0); @@ -958,6 +1100,14 @@ namespace db if (!requests_stream) return requests_stream.error(); + const auto ranges_data = subaddress_ranges.get_all(*ranges_cur); + if (!ranges_data) + return ranges_data.error(); + + auto indexes_stream = subaddress_indexes.get_key_stream(std::move(indexes_cur)); + if (!indexes_stream) + return indexes_stream.error(); + // This list should be smaller ... ? const auto webhooks_data = webhooks.get_all(*webhooks_cur); if (!webhooks_data) @@ -978,6 +1128,8 @@ namespace db wire::field(spends.name, wire::as_object(spends_stream->make_range(), wire::as_integer, wire::as_array)), wire::field(images.name, wire::as_object(images_stream->make_range(), output_id_key{}, wire::as_array)), wire::field(requests.name, wire::as_object(requests_stream->make_range(), wire::enum_as_string, toggle_keys_filter)), + wire::field(subaddress_ranges.name, std::cref(*ranges_data)), + wire::field(subaddress_indexes.name, wire::as_object(indexes_stream->make_range(), wire::as_integer, wire::as_array)), wire::field(webhooks.name, std::cref(*webhooks_data)), wire::field(events_by_account_id.name, wire::as_object(events_stream->make_range(), wire::as_integer, wire::as_array)) ); @@ -2215,6 +2367,173 @@ namespace db }); } + expect> + storage::upsert_subaddresses(const account_id id, const account_address& address, const crypto::secret_key& view_key, std::vector subaddrs, const std::uint32_t max_subaddr) + { + MONERO_PRECOND(db != nullptr); + std::sort(subaddrs.begin(), subaddrs.end()); + + return db->try_write([this, id, &address, &view_key, &subaddrs, max_subaddr] (MDB_txn& txn) -> expect> + { + std::size_t subaddr_count = 0; + std::vector out{}; + index_ranges new_dict{}; + const auto add_out = [&out] (major_index major, index_range minor) + { + if (out.empty() || out.back().first != major) + out.emplace_back(major, index_ranges{minor}); + else + out.back().second.push_back(minor); + }; + + const auto check_max_range = [&subaddr_count, max_subaddr] (const index_range& range) -> bool + { + const auto more = std::uint32_t(range[1]) - std::uint32_t(range[0]); + if (max_subaddr - subaddr_count <= more) + return false; + subaddr_count += more + 1; + return true; + }; + const auto check_max_ranges = [&check_max_range] (const index_ranges& ranges) -> bool + { + for (const auto& range : ranges) + { + if (!check_max_range(range)) + return false; + } + return true; + }; + + cursor::subaddress_ranges ranges_cur; + cursor::subaddress_indexes indexes_cur; + + MONERO_CHECK(check_cursor(txn, this->db->tables.subaddress_ranges, ranges_cur)); + MONERO_CHECK(check_cursor(txn, this->db->tables.subaddress_indexes, indexes_cur)); + + MDB_val key = lmdb::to_val(id); + MDB_val value{}; + int err = mdb_cursor_get(indexes_cur.get(), &key, &value, MDB_SET); + if (err) + { + if (err != MDB_NOTFOUND) + return {lmdb::error(err)}; + } + else + { + MONERO_LMDB_CHECK(mdb_cursor_count(indexes_cur.get(), &subaddr_count)); + if (max_subaddr < subaddr_count) + return {error::max_subaddresses}; + } + + for (auto& major_entry : subaddrs) + { + new_dict.clear(); + if (!check_subaddress_dict(major_entry)) + { + MERROR("Invalid subaddress_dict given to storage::upsert_subaddrs"); + return {wire::error::schema::array}; + } + + value = lmdb::to_val(major_entry.first); + err = mdb_cursor_get(ranges_cur.get(), &key, &value, MDB_GET_BOTH); + if (err) + { + if (err != MDB_NOTFOUND) + return {lmdb::error(err)}; + if (!check_max_ranges(major_entry.second)) + return {error::max_subaddresses}; + out.push_back(major_entry); + new_dict = std::move(major_entry.second); + } + else // merge new minor index ranges with old + { + auto old_dict = subaddress_ranges.get_value(value); + if (!old_dict) + return old_dict.error(); + + auto& old_range = old_dict->second; + const auto& new_range = major_entry.second; + + auto old_loc = old_range.begin(); + auto new_loc = new_range.begin(); + for ( ; old_loc != old_range.end() && new_loc != new_range.end(); ) + { + if (std::uint64_t(new_loc->at(1)) + 1 < std::uint32_t(old_loc->at(0))) + { // new has no overlap with existing + if (!check_max_range(*new_loc)) + return {error::max_subaddresses}; + + new_dict.push_back(*new_loc); + add_out(major_entry.first, *new_loc); + ++new_loc; + } + else if (std::uint64_t(old_loc->at(1)) + 1 < std::uint32_t(new_loc->at(0))) + { // existing has no overlap with new + new_dict.push_back(*old_loc); + ++old_loc; + } + else if (old_loc->at(0) <= new_loc->at(0) && new_loc->at(1) <= old_loc->at(1)) + { // new is completely within existing + ++new_loc; + } + else // new overlap at beginning, end, or both + { + if (new_loc->at(0) < old_loc->at(0)) + { // overlap at beginning + const index_range new_range{new_loc->at(0), minor_index(std::uint32_t(old_loc->at(0)) - 1)}; + if (!check_max_range(new_range)) + return {error::max_subaddresses}; + add_out(major_entry.first, new_range); + old_loc->at(0) = new_loc->at(0); + } + if (old_loc->at(1) < new_loc->at(1)) + { // overlap at end + const index_range new_range{minor_index(std::uint32_t(old_loc->at(1)) + 1), new_loc->at(1)}; + if (!check_max_range(new_range)) + return {error::max_subaddresses}; + add_out(major_entry.first, new_range); + old_loc->at(1) = new_loc->at(1); + } + ++new_loc; + } + } + + std::copy(old_loc, old_range.end(), std::back_inserter(new_dict)); + for ( ; new_loc != new_range.end(); ++new_loc) + { + if (!check_max_range(*new_loc)) + return {error::max_subaddresses}; + new_dict.push_back(*new_loc); + add_out(major_entry.first, *new_loc); + } + } + + for (const auto& new_indexes : new_dict) + { + for (std::uint64_t minor : boost::counting_range(std::uint64_t(new_indexes[0]), std::uint64_t(new_indexes[1]) + 1)) + { + subaddress_map new_value{}; + new_value.index = address_index{major_entry.first, minor_index(minor)}; + new_value.subaddress = new_value.index.get_spend_public(address, view_key); + + value = lmdb::to_val(new_value); + + MONERO_LMDB_CHECK(mdb_cursor_put(indexes_cur.get(), &key, &value, 0)); + } + } + + const expect value_bytes = + subaddress_ranges.make_value(major_entry.first, new_dict); + if (!value_bytes) + return value_bytes.error(); + value = MDB_val{value_bytes->size(), const_cast(static_cast(value_bytes->data()))}; + MONERO_LMDB_CHECK(mdb_cursor_put(ranges_cur.get(), &key, &value, 0)); + } + + return {std::move(out)}; + }); + } + expect storage::add_webhook(const webhook_type type, const boost::optional& address, const webhook_value& event) { if (event.second.url != "zmq") @@ -2253,8 +2572,10 @@ namespace db return {error::bad_webhook}; lmkey = lmdb::to_val(key); - const epee::byte_slice value = webhooks.make_value(event.first, event.second); - lmvalue = MDB_val{value.size(), const_cast(static_cast(value.data()))}; + const expect value = webhooks.make_value(event.first, event.second); + if (!value) + return value.error(); + lmvalue = MDB_val{value->size(), const_cast(static_cast(value->data()))}; MONERO_LMDB_CHECK(mdb_cursor_put(webhooks_cur.get(), &lmkey, &lmvalue, 0)); return success(); }); diff --git a/src/db/storage.h b/src/db/storage.h index 396bef4..43e2446 100644 --- a/src/db/storage.h +++ b/src/db/storage.h @@ -52,6 +52,8 @@ namespace db MONERO_CURSOR(spends); MONERO_CURSOR(images); MONERO_CURSOR(requests); + MONERO_CURSOR(subaddress_ranges); + MONERO_CURSOR(subaddress_indexes); MONERO_CURSOR(blocks); MONERO_CURSOR(accounts_by_address); @@ -133,6 +135,13 @@ namespace db expect get_request(request type, account_address const& address, cursor::requests cur = nullptr) noexcept; + //! \return All subaddresses activated for account `id`. + expect> get_subaddresses(account_id id, cursor::subaddress_ranges cur = nullptr) noexcept; + + //! \return A specific subaddress index + expect + find_subaddress(account_id id, crypto::public_key const& spend_public, cursor::subaddress_indexes& cur) noexcept; + //! \return All webhook values associated with user `key` and `payment_id`. expect> find_webhook(webhook_key const& key, crypto::hash8 const& payment_id, cursor::webhooks cur = nullptr); @@ -243,6 +252,24 @@ namespace db expect>> update(block_id height, epee::span chain, epee::span accts); + /*! + Adds subaddresses to an account. Upon success, an account will + immediately begin tracking them in the scanner. + + \param id of the account to associate new indexes + \param addresss of the account (needed to generate subaddress publc key) + \param view_key of the account (needed to generate subaddress public key) + \param subaddrs Range of subaddress indexes that need to be added to the + database. Indexes _may_ overlap with existing indexes. + \param max_subaddresses The maximum number of subaddresses allowed per + account. + + \return The new ranges of subaddress indexes added to the database + (whereas `subaddrs` may overlap with existing indexes). + */ + expect> + upsert_subaddresses(account_id id, const account_address& address, const crypto::secret_key& view_key, std::vector subaddrs, std::uint32_t max_subaddresses); + /*! Add webhook to be tracked in the database. The webhook will "call" the specified URL with JSON/msgpack information when the event occurs. diff --git a/src/error.cpp b/src/error.cpp index e5d4642..7f573fd 100644 --- a/src/error.cpp +++ b/src/error.cpp @@ -85,10 +85,14 @@ namespace lws return "Unspecified error when retrieving exchange rates"; case error::http_server: return "HTTP server failed"; + case error::invalid_range: + return "Invalid subaddress range provided"; case error::json_rpc: return "Error returned by JSON-RPC server"; case error::exchange_rates_old: return "Exchange rates are older than cache interval"; + case error::max_subaddresses: + return "Max subaddresses exceeded"; case error::not_enough_mixin: return "Not enough outputs to meet requested mixin count"; case error::signal_abort_process: diff --git a/src/error.h b/src/error.h index c243c28..7d6c4e8 100644 --- a/src/error.h +++ b/src/error.h @@ -57,7 +57,9 @@ namespace lws exchange_rates_fetch, //!< Exchange rates fetching failed exchange_rates_old, //!< Exchange rates are older than cache interval http_server, //!< HTTP server failure (init or run) + invalid_range, //!< Invalid subaddress range provided json_rpc, //!< Error returned by JSON-RPC server + max_subaddresses, //!< Max subaddresses exceeded not_enough_mixin, //!< Not enough outputs to meet mixin count signal_abort_process, //!< In process ZMQ PUB to abort the process was received signal_abort_scan, //!< In process ZMQ PUB to abort the scan was received diff --git a/src/lmdb/msgpack_table.h b/src/lmdb/msgpack_table.h index 239dceb..4c70abe 100644 --- a/src/lmdb/msgpack_table.h +++ b/src/lmdb/msgpack_table.h @@ -33,13 +33,20 @@ namespace lmdb return out; } - static epee::byte_slice make_value(const fixed_value_type& val1, const msgpack_value_type& val2) + static expect make_value(const fixed_value_type& val1, const msgpack_value_type& val2) { epee::byte_stream initial; initial.write({reinterpret_cast(std::addressof(val1)), sizeof(val1)}); wire::msgpack_slice_writer dest{std::move(initial), true}; - wire_write::bytes(dest, val2); + try + { + wire_write::bytes(dest, val2); + } + catch (const wire::exception& e) + { + return {e.code()}; + } return epee::byte_slice{dest.take_sink()}; } @@ -80,12 +87,9 @@ namespace lmdb auto msgpack_bytes = lmdb::to_byte_span(value); msgpack_bytes.remove_prefix(sizeof(out.first)); - msgpack_value_type second{}; - const std::error_code error = wire::msgpack::from_bytes(epee::byte_slice{{msgpack_bytes}}, second); + const std::error_code error = wire::msgpack::from_bytes(epee::byte_slice{{msgpack_bytes}}, out.second); if (error) return error; - out.second = std::move(second); - return out; } diff --git a/src/rest_server.cpp b/src/rest_server.cpp index 814c955..5f9e877 100644 --- a/src/rest_server.cpp +++ b/src/rest_server.cpp @@ -27,6 +27,7 @@ #include "rest_server.h" #include +#include #include #include #include @@ -140,6 +141,7 @@ namespace lws struct runtime_options { + std::uint32_t max_subaddresses; epee::net_utils::ssl_verification_t webhook_verify; bool disable_admin_auth; }; @@ -482,6 +484,23 @@ namespace lws } }; + struct get_subaddrs + { + using request = rpc::account_credentials; + using response = rpc::get_subaddrs_response; + + static expect handle(request const& req, db::storage disk, rpc::client const&, runtime_options const& options) + { + auto user = open_account(req, std::move(disk)); + if (!user) + return user.error(); + auto subaddrs = user->second.get_subaddresses(user->first.id); + if (!subaddrs) + return subaddrs.error(); + return response{std::move(*subaddrs)}; + } + }; + struct get_unspent_outs { using request = rpc::get_unspent_outs_request; @@ -642,6 +661,68 @@ namespace lws } }; + struct provision_subaddrs + { + using request = rpc::provision_subaddrs_request; + using response = rpc::new_subaddrs_response; + + static expect handle(request req, db::storage disk, rpc::client const&, runtime_options const& options) + { + if (!req.maj_i && !req.min_i && !req.n_min && !req.n_maj) + return {lws::error::invalid_range}; + + db::account_id id = db::account_id::invalid; + { + auto user = open_account(req.creds, disk.clone()); + if (!user) + return user.error(); + id = user->first.id; + } + + const std::uint32_t major_i = req.maj_i.value_or(0); + const std::uint32_t minor_i = req.min_i.value_or(0); + const std::uint32_t n_major = req.n_maj.value_or(50); + const std::uint32_t n_minor = req.n_min.value_or(500); + const bool get_all = req.get_all.value_or(true); + + if (std::numeric_limits::max() / n_major < n_minor) + return {lws::error::max_subaddresses}; + if (options.max_subaddresses < n_major * n_minor) + return {lws::error::max_subaddresses}; + + std::vector new_ranges; + std::vector all_ranges; + if (n_major && n_minor) + { + std::vector ranges; + ranges.reserve(n_major); + for (std::uint64_t elem : boost::counting_range(std::uint64_t(major_i), std::uint64_t(major_i) + n_major)) + { + ranges.emplace_back( + db::major_index(elem), db::index_ranges{db::index_range{db::minor_index(minor_i), db::minor_index(minor_i + n_minor - 1)}} + ); + } + auto upserted = disk.upsert_subaddresses(id, req.creds.address, req.creds.key, ranges, options.max_subaddresses); + if (!upserted) + return upserted.error(); + new_ranges = std::move(*upserted); + } + + if (get_all) + { + // must start a new read after the last write + auto reader = disk.start_read(); + if (!reader) + return reader.error(); + auto rc = reader->get_subaddresses(id); + if (!rc) + return rc.error(); + all_ranges = std::move(*rc); + } + return response{std::move(new_ranges), std::move(all_ranges)}; + } + }; + struct submit_raw_tx { using request = rpc::submit_raw_tx_request; @@ -672,6 +753,46 @@ namespace lws } }; + struct upsert_subaddrs + { + using request = rpc::upsert_subaddrs_request; + using response = rpc::new_subaddrs_response; + + static expect handle(request req, db::storage disk, rpc::client const&, runtime_options const& options) + { + if (!options.max_subaddresses) + return {lws::error::max_subaddresses}; + + db::account_id id = db::account_id::invalid; + { + auto user = open_account(req.creds, disk.clone()); + if (!user) + return user.error(); + id = user->first.id; + } + + const bool get_all = req.get_all.value_or(true); + + std::vector all_ranges; + auto new_ranges = + disk.upsert_subaddresses(id, req.creds.address, req.creds.key, req.subaddrs, options.max_subaddresses); + if (!new_ranges) + return new_ranges.error(); + + if (get_all) + { + auto reader = disk.start_read(); + if (!reader) + return reader.error(); + auto rc = reader->get_subaddresses(id); + if (!rc) + return rc.error(); + all_ranges = std::move(*rc); + } + return response{std::move(*new_ranges), std::move(all_ranges)}; + } + }; + template expect call(std::string&& root, db::storage disk, const rpc::client& gclient, const runtime_options& options) { @@ -758,14 +879,17 @@ namespace lws constexpr const endpoint endpoints[] = { - {"/get_address_info", call, 2 * 1024}, - {"/get_address_txs", call, 2 * 1024}, - {"/get_random_outs", call, 2 * 1024}, - {"/get_txt_records", nullptr, 0 }, - {"/get_unspent_outs", call, 2 * 1024}, - {"/import_wallet_request", call, 2 * 1024}, - {"/login", call, 2 * 1024}, - {"/submit_raw_tx", call, 50 * 1024} + {"/get_address_info", call, 2 * 1024}, + {"/get_address_txs", call, 2 * 1024}, + {"/get_random_outs", call, 2 * 1024}, + {"/get_subaddrs", call, 2 * 1024}, + {"/get_txt_records", nullptr, 0 }, + {"/get_unspent_outs", call, 2 * 1024}, + {"/import_wallet_request", call, 2 * 1024}, + {"/login", call, 2 * 1024}, + {"/provision_subaddrs", call, 2 * 1024}, + {"/submit_raw_tx", call, 50 * 1024}, + {"/upsert_subaddrs", call, 10 * 1024} }; constexpr const endpoint admin_endpoints[] = @@ -893,7 +1017,7 @@ namespace lws { MINFO(body.error().message() << " from " << ctx.m_remote_address.str() << " on " << handler->name); - if (body.error().category() == wire::error::rapidjson_category()) + if (body.error().category() == wire::error::rapidjson_category() || body == lws::error::invalid_range) { response.m_response_code = 400; response.m_response_comment = "Bad Request"; @@ -903,6 +1027,11 @@ namespace lws response.m_response_code = 403; response.m_response_comment = "Forbidden"; } + else if (body == lws::error::max_subaddresses) + { + response.m_response_code = 409; + response.m_response_comment = "Conflict"; + } else if (body.matches(std::errc::timed_out) || body.matches(std::errc::no_lock_available)) { response.m_response_code = 503; @@ -1017,7 +1146,7 @@ namespace lws }; bool any_ssl = false; - const runtime_options options{config.webhook_verify, config.disable_admin_auth}; + const runtime_options options{config.max_subaddresses, config.webhook_verify, config.disable_admin_auth}; for (const std::string& address : addresses) { ports_.emplace_back(io_service_, disk.clone(), MONERO_UNWRAP(client.clone()), options); diff --git a/src/rest_server.h b/src/rest_server.h index df02709..36dc068 100644 --- a/src/rest_server.h +++ b/src/rest_server.h @@ -53,6 +53,7 @@ namespace lws epee::net_utils::ssl_authentication_t auth; std::vector access_controls; std::size_t threads; + std::uint32_t max_subaddresses; epee::net_utils::ssl_verification_t webhook_verify; bool allow_external; bool disable_admin_auth; diff --git a/src/rpc/light_wallet.cpp b/src/rpc/light_wallet.cpp index 64cd0dc..6b36575 100644 --- a/src/rpc/light_wallet.cpp +++ b/src/rpc/light_wallet.cpp @@ -134,7 +134,8 @@ namespace wire::field("timestamp", iso_timestamp(self.data.first.timestamp)), wire::field("height", self.data.first.link.height), wire::field("spend_key_images", std::cref(self.data.second)), - wire::optional_field("rct", optional_rct) + wire::optional_field("rct", optional_rct), + wire::field("recipient", std::cref(self.data.first.recipient)) ); } @@ -192,6 +193,12 @@ namespace lws convert_address(address, self.address); } + void rpc::write_bytes(wire::json_writer& dest, const new_subaddrs_response& self) + { + wire::object(dest, WIRE_FIELD(new_subaddrs), WIRE_FIELD(all_subaddrs)); + } + + void rpc::write_bytes(wire::json_writer& dest, const transaction_spend& self) { wire::object(dest, @@ -199,7 +206,8 @@ namespace lws wire::field("key_image", std::cref(self.possible_spend.image)), wire::field("tx_pub_key", std::cref(self.meta.tx_public)), wire::field("out_index", self.meta.index), - wire::field("mixin", self.possible_spend.mixin_count) + wire::field("mixin", self.possible_spend.mixin_count), + wire::field("sender", std::cref(self.possible_spend.sender)) ); } @@ -278,6 +286,11 @@ namespace lws wire::object(dest, WIRE_FIELD(amount_outs)); } + void rpc::write_bytes(wire::json_writer& dest, const get_subaddrs_response& self) + { + wire::object(dest, WIRE_FIELD(all_subaddrs)); + } + void rpc::read_bytes(wire::json_reader& source, get_unspent_outs_request& self) { std::string address; @@ -331,6 +344,21 @@ namespace lws wire::object(dest, WIRE_FIELD_COPY(new_address), WIRE_FIELD_COPY(generated_locally)); } + void rpc::read_bytes(wire::json_reader& source, provision_subaddrs_request& self) + { + std::string address; + wire::object(source, + wire::field("address", std::ref(address)), + wire::field("view_key", std::ref(unwrap(unwrap(self.creds.key)))), + WIRE_OPTIONAL_FIELD(maj_i), + WIRE_OPTIONAL_FIELD(min_i), + WIRE_OPTIONAL_FIELD(n_maj), + WIRE_OPTIONAL_FIELD(n_min), + WIRE_OPTIONAL_FIELD(get_all) + ); + convert_address(address, self.creds.address); + } + void rpc::read_bytes(wire::json_reader& source, submit_raw_tx_request& self) { wire::object(source, WIRE_FIELD(tx)); @@ -339,4 +367,16 @@ namespace lws { wire::object(dest, WIRE_FIELD_COPY(status)); } + + void rpc::read_bytes(wire::json_reader& source, upsert_subaddrs_request& self) + { + std::string address; + wire::object(source, + wire::field("address", std::ref(address)), + wire::field("view_key", std::ref(unwrap(unwrap(self.creds.key)))), + WIRE_FIELD(subaddrs), + WIRE_OPTIONAL_FIELD(get_all) + ); + convert_address(address, self.creds.address); + } } // lws diff --git a/src/rpc/light_wallet.h b/src/rpc/light_wallet.h index c38a6f7..9e9708e 100644 --- a/src/rpc/light_wallet.h +++ b/src/rpc/light_wallet.h @@ -65,6 +65,15 @@ namespace rpc void read_bytes(wire::json_reader&, account_credentials&); + struct new_subaddrs_response + { + new_subaddrs_response() = delete; + std::vector new_subaddrs; + std::vector all_subaddrs; + }; + void write_bytes(wire::json_writer&, const new_subaddrs_response&); + + struct transaction_spend { transaction_spend() = delete; @@ -164,6 +173,14 @@ namespace rpc void write_bytes(wire::json_writer&, const get_unspent_outs_response&); + struct get_subaddrs_response + { + get_subaddrs_response() = delete; + std::vector all_subaddrs; + }; + void write_bytes(wire::json_writer&, const get_subaddrs_response&); + + struct import_response { import_response() = delete; @@ -193,6 +210,19 @@ namespace rpc void write_bytes(wire::json_writer&, login_response); + struct provision_subaddrs_request + { + provision_subaddrs_request() = delete; + account_credentials creds; + boost::optional maj_i; + boost::optional min_i; + boost::optional n_maj; + boost::optional n_min; + boost::optional get_all; + }; + void read_bytes(wire::json_reader&, provision_subaddrs_request&); + + struct submit_raw_tx_request { submit_raw_tx_request() = delete; @@ -206,5 +236,15 @@ namespace rpc const char* status; }; void write_bytes(wire::json_writer&, submit_raw_tx_response); + + + struct upsert_subaddrs_request + { + upsert_subaddrs_request() = delete; + account_credentials creds; + std::vector subaddrs; + boost::optional get_all; + }; + void read_bytes(wire::json_reader&, upsert_subaddrs_request&); } // rpc } // lws diff --git a/src/scanner.cpp b/src/scanner.cpp index 086e7bb..1533eed 100644 --- a/src/scanner.cpp +++ b/src/scanner.cpp @@ -94,16 +94,22 @@ namespace lws std::atomic update; }; + struct options + { + net::ssl_verification_t webhook_verify; + bool enable_subaddresses; + }; + struct thread_data { - explicit thread_data(rpc::client client, db::storage disk, std::vector users, net::ssl_verification_t webhook_verify) - : client(std::move(client)), disk(std::move(disk)), users(std::move(users)), webhook_verify(webhook_verify) + explicit thread_data(rpc::client client, db::storage disk, std::vector users, options opts) + : client(std::move(client)), disk(std::move(disk)), users(std::move(users)), opts(opts) {} rpc::client client; db::storage disk; std::vector users; - net::ssl_verification_t webhook_verify; + options opts; }; // until we have a signal-handler safe notification system @@ -263,6 +269,23 @@ namespace lws } }; + struct subaddress_reader + { + expect reader; + db::cursor::subaddress_indexes cur; + + subaddress_reader(db::storage const& disk, const bool enable_subaddresses) + : reader(common_error::kInvalidArgument), cur(nullptr) + { + if (enable_subaddresses) + { + reader = disk.start_read(); + if (!reader) + MERROR("Subadress lookup failure: " << reader.error().message()); + } + } + }; + void scan_transaction_base( epee::span users, const db::block_id height, @@ -270,6 +293,7 @@ namespace lws crypto::hash const& tx_hash, cryptonote::transaction const& tx, std::vector const& out_ids, + subaddress_reader& reader, std::function spend_action, std::function output_action) { @@ -280,6 +304,8 @@ namespace lws boost::optional prefix_hash; boost::optional extra_nonce; std::pair payment_id; + cryptonote::tx_extra_additional_pub_keys additional_tx_pub_keys; + std::vector additional_derivations; { std::vector extra; @@ -297,6 +323,10 @@ namespace lws } else extra_nonce = boost::none; + + // additional tx pub keys present when there are 3+ outputs in a tx involving subaddresses + if (reader.reader) + cryptonote::find_tx_extra_field_by_type(extra, additional_tx_pub_keys); } // destruct `extra` vector for (account& user : users) @@ -308,6 +338,21 @@ namespace lws if (!crypto::wallet::generate_key_derivation(key.pub_key, user.view_key(), derived)) continue; // to next user + if (reader.reader && additional_tx_pub_keys.data.size() == tx.vout.size()) + { + additional_derivations.resize(tx.vout.size()); + std::size_t index = -1; + for (auto const& out: tx.vout) + { + ++index; + if (!crypto::wallet::generate_key_derivation(additional_tx_pub_keys.data[index], user.view_key(), additional_derivations[index])) + { + additional_derivations.clear(); + break; // vout loop + } + } + } + db::extra ext{}; std::uint32_t mixin = 0; for (auto const& in : tx.vin) @@ -324,23 +369,26 @@ namespace lws for (std::uint64_t offset : in_data->key_offsets) { goffset += offset; - if (user.has_spendable(db::output_id{in_data->amount, goffset})) - { - spend_action( - user, - db::spend{ - db::transaction_link{height, tx_hash}, - in_data->k_image, - db::output_id{in_data->amount, goffset}, - timestamp, - tx.unlock_time, - mixin, - {0, 0, 0}, // reserved - payment_id.first, - payment_id.second.long_ - } - ); - } + const boost::optional subaccount = + user.get_spendable(db::output_id{in_data->amount, goffset}); + if (!subaccount) + continue; // to next input + + spend_action( + user, + db::spend{ + db::transaction_link{height, tx_hash}, + in_data->k_image, + db::output_id{in_data->amount, goffset}, + timestamp, + tx.unlock_time, + mixin, + {0, 0, 0}, // reserved + payment_id.first, + payment_id.second.long_, + *subaccount + } + ); } } else if (boost::get(std::addressof(in))) @@ -358,15 +406,57 @@ namespace lws boost::optional view_tag_opt = cryptonote::get_output_view_tag(out); - if (!cryptonote::out_can_be_to_acc(view_tag_opt, derived, index)) + + const bool found_tag = + (!additional_derivations.empty() && cryptonote::out_can_be_to_acc(view_tag_opt, additional_derivations.at(index), index)) || + cryptonote::out_can_be_to_acc(view_tag_opt, derived, index); + + if (!found_tag) continue; // to next output - crypto::public_key derived_pub; - const bool received = - crypto::wallet::derive_subaddress_public_key(out_pub_key, derived, index, derived_pub) && - derived_pub == user.spend_public(); + bool found_pub = false; + db::address_index account_index{db::major_index::primary, db::minor_index::primary}; + crypto::key_derivation active_derived; - if (!received) + // inspect the additional and traditional keys + for (std::size_t attempt = 0; attempt < 2; ++attempt) + { + if (attempt == 0) + active_derived = derived; + else if (!additional_derivations.empty()) + active_derived = additional_derivations.at(index); + else + break; // inspection loop + + crypto::public_key derived_pub; + if (!crypto::wallet::derive_subaddress_public_key(out_pub_key, active_derived, index, derived_pub)) + continue; // to next available active_derived + + if (user.spend_public() != derived_pub) + { + if (!reader.reader) + continue; // to next available active_derived + + const expect match = + reader.reader->find_subaddress(user.id(), derived_pub, reader.cur); + if (!match) + { + if (match != lmdb::error(MDB_NOTFOUND)) + MERROR("Failure when doing subaddress search: " << match.error().message()); + continue; // to next available active_derived + } + found_pub = true; + account_index = *match; + break; // additional_derivations loop + } + else + { + found_pub = true; + break; // additional_derivations loop + } + } + + if (!found_pub) continue; // to next output if (!prefix_hash) @@ -381,7 +471,7 @@ namespace lws { const bool bulletproof2 = (rct::RCTTypeBulletproof2 <= tx.rct_signatures.type); const auto decrypted = lws::decode_amount( - tx.rct_signatures.outPk.at(index).mask, tx.rct_signatures.ecdhInfo.at(index), derived, index, bulletproof2 + tx.rct_signatures.outPk.at(index).mask, tx.rct_signatures.ecdhInfo.at(index), active_derived, index, bulletproof2 ); if (!decrypted) { @@ -398,7 +488,7 @@ namespace lws if (!payment_id.first && cryptonote::get_encrypted_payment_id_from_tx_extra_nonce(extra_nonce->nonce, payment_id.second.short_)) { payment_id.first = sizeof(crypto::hash8); - lws::decrypt_payment_id(payment_id.second.short_, derived); + lws::decrypt_payment_id(payment_id.second.short_, active_derived); } } @@ -421,7 +511,8 @@ namespace lws {0, 0, 0, 0, 0, 0, 0}, // reserved bytes db::pack(ext, payment_id.first), payment_id.second, - tx.rct_signatures.txnFee + tx.rct_signatures.txnFee, + account_index } ); @@ -437,12 +528,13 @@ namespace lws const std::uint64_t timestamp, crypto::hash const& tx_hash, cryptonote::transaction const& tx, - std::vector const& out_ids) + std::vector const& out_ids, + subaddress_reader& reader) { - scan_transaction_base(users, height, timestamp, tx_hash, tx, out_ids, add_spend{}, add_output{}); + scan_transaction_base(users, height, timestamp, tx_hash, tx, out_ids, reader, add_spend{}, add_output{}); } - void scan_transactions(std::string&& txpool_msg, epee::span users, db::storage const& disk, rpc::client& client, const net::ssl_verification_t verify_mode) + void scan_transactions(std::string&& txpool_msg, epee::span users, db::storage const& disk, rpc::client& client, const options& opts) { // uint64::max is for txpool static const std::vector fake_outs( @@ -459,9 +551,10 @@ namespace lws const auto time = boost::numeric_cast(std::chrono::system_clock::to_time_t(std::chrono::system_clock::now())); - send_webhook sender{disk, client, verify_mode}; + subaddress_reader reader{disk, opts.enable_subaddresses}; + send_webhook sender{disk, client, opts.webhook_verify}; for (const auto& tx : parsed->txes) - scan_transaction_base(users, db::block_id::txpool, time, crypto::hash{}, tx, fake_outs, null_spend{}, sender); + scan_transaction_base(users, db::block_id::txpool, time, crypto::hash{}, tx, fake_outs, reader, null_spend{}, sender); } void update_rates(rpc::context& ctx) @@ -481,7 +574,7 @@ namespace lws rpc::client client{std::move(data->client)}; db::storage disk{std::move(data->disk)}; std::vector users{std::move(data->users)}; - const net::ssl_verification_t webhook_verify = data->webhook_verify; + const options opts = std::move(data->opts); assert(!users.empty()); assert(std::is_sorted(users.begin(), users.end(), by_height{})); @@ -568,7 +661,7 @@ namespace lws { if (message->first != rpc::client::topic::txpool) break; // inner for loop - scan_transactions(std::move(message->second), epee::to_mut_span(users), disk, client, webhook_verify); + scan_transactions(std::move(message->second), epee::to_mut_span(users), disk, client, opts); } for ( ; message != new_pubs->end(); ++message) @@ -605,6 +698,7 @@ namespace lws else fetched->start_height = 0; + subaddress_reader reader{disk, opts.enable_subaddresses}; for (auto block_data : boost::combine(blocks, indices)) { ++(fetched->start_height); @@ -629,7 +723,8 @@ namespace lws block.timestamp, miner_tx_hash, block.miner_tx, - *(indices.begin()) + *(indices.begin()), + reader ); indices.remove_prefix(1); @@ -644,13 +739,15 @@ namespace lws block.timestamp, boost::get<0>(tx_data), boost::get<1>(tx_data), - boost::get<2>(tx_data) + boost::get<2>(tx_data), + reader ); } blockchain.push_back(cryptonote::get_block_hash(block)); } // for each block + reader.reader = std::error_code{common_error::kInvalidArgument}; // cleanup reader before next write auto updated = disk.update( users.front().scan_height(), epee::to_span(blockchain), epee::to_span(users) ); @@ -665,7 +762,7 @@ namespace lws } MINFO("Processed " << blocks.size() << " block(s) against " << users.size() << " account(s)"); - send_payment_hook(client, epee::to_span(updated->second), webhook_verify); + send_payment_hook(client, epee::to_span(updated->second), opts.webhook_verify); if (updated->first != users.size()) { MWARNING("Only updated " << updated->first << " account(s) out of " << users.size() << ", resetting"); @@ -692,7 +789,7 @@ namespace lws Launches `thread_count` threads to run `scan_loop`, and then polls for active account changes in background */ - void check_loop(db::storage disk, rpc::context& ctx, std::size_t thread_count, std::vector users, std::vector active, const net::ssl_verification_t webhook_verify) + void check_loop(db::storage disk, rpc::context& ctx, std::size_t thread_count, std::vector users, std::vector active, const options opts) { assert(0 < thread_count); assert(0 < users.size()); @@ -754,7 +851,7 @@ namespace lws client.watch_scan_signals(); auto data = std::make_shared( - std::move(client), disk.clone(), std::move(thread_users), webhook_verify + std::move(client), disk.clone(), std::move(thread_users), opts ); threads.emplace_back(attrs, std::bind(&scan_loop, std::ref(self), std::move(data))); } @@ -765,7 +862,7 @@ namespace lws client.watch_scan_signals(); auto data = std::make_shared( - std::move(client), disk.clone(), std::move(users), webhook_verify + std::move(client), disk.clone(), std::move(users), opts ); threads.emplace_back(attrs, std::bind(&scan_loop, std::ref(self), std::move(data))); } @@ -908,7 +1005,7 @@ namespace lws return {std::move(client)}; } - void scanner::run(db::storage disk, rpc::context ctx, std::size_t thread_count, const epee::net_utils::ssl_verification_t webhook_verify) + void scanner::run(db::storage disk, rpc::context ctx, std::size_t thread_count, const epee::net_utils::ssl_verification_t webhook_verify, const bool enable_subaddresses) { thread_count = std::max(std::size_t(1), thread_count); @@ -931,7 +1028,7 @@ namespace lws for (db::account user : accounts.make_range()) { - std::vector receives{}; + std::vector> receives{}; std::vector pubs{}; auto receive_list = MONERO_UNWRAP(reader.get_outputs(user.id)); @@ -941,7 +1038,9 @@ namespace lws for (auto output = receive_list.make_iterator(); !output.is_end(); ++output) { - receives.emplace_back(output.get_value()); + auto id = output.get_value(); + auto subaddr = output.get_value(); + receives.emplace_back(std::move(id), std::move(subaddr)); pubs.emplace_back(output.get_value()); } @@ -960,7 +1059,7 @@ namespace lws checked_wait(account_poll_interval - (std::chrono::steady_clock::now() - last)); } else - check_loop(disk.clone(), ctx, thread_count, std::move(users), std::move(active), webhook_verify); + check_loop(disk.clone(), ctx, thread_count, std::move(users), std::move(active), options{webhook_verify, enable_subaddresses}); if (!scanner::is_running()) return; diff --git a/src/scanner.h b/src/scanner.h index 55fba96..bccfeb0 100644 --- a/src/scanner.h +++ b/src/scanner.h @@ -49,7 +49,7 @@ namespace lws static expect sync(db::storage disk, rpc::client client); //! Poll daemon until `stop()` is called, using `thread_count` threads. - static void run(db::storage disk, rpc::context ctx, std::size_t thread_count, epee::net_utils::ssl_verification_t webhook_verify); + static void run(db::storage disk, rpc::context ctx, std::size_t thread_count, epee::net_utils::ssl_verification_t webhook_verify, bool enable_subaddresses); //! \return True if `stop()` has never been called. static bool is_running() noexcept { return running; } diff --git a/src/server_main.cpp b/src/server_main.cpp index 7348fe0..7e62af8 100644 --- a/src/server_main.cpp +++ b/src/server_main.cpp @@ -78,6 +78,7 @@ namespace const command_line::arg_descriptor disable_admin_auth; const command_line::arg_descriptor webhook_ssl_verification; const command_line::arg_descriptor config_file; + const command_line::arg_descriptor max_subaddresses; static std::string get_default_zmq() { @@ -120,6 +121,7 @@ namespace , disable_admin_auth{"disable-admin-auth", "Make auth field optional in HTTP-REST requests", false} , webhook_ssl_verification{"webhook-ssl-verification", "[] specify SSL verification mode for webhooks", "system_ca"} , config_file{"config-file", "Specify any option in a config file; = on separate lines"} + , max_subaddresses{"max-subaddresses", "Maximum number of subaddresses per primary account (defaults to 0)", 0} {} void prepare(boost::program_options::options_description& description) const @@ -150,6 +152,7 @@ namespace command_line::add_arg(description, disable_admin_auth); command_line::add_arg(description, webhook_ssl_verification); command_line::add_arg(description, config_file); + command_line::add_arg(description, max_subaddresses); } }; @@ -230,6 +233,7 @@ namespace {command_line::get_arg(args, opts.rest_ssl_key), command_line::get_arg(args, opts.rest_ssl_cert)}, command_line::get_arg(args, opts.access_controls), command_line::get_arg(args, opts.rest_threads), + command_line::get_arg(args, opts.max_subaddresses), webhook_verify, command_line::get_arg(args, opts.external_bind), command_line::get_arg(args, opts.disable_admin_auth) @@ -250,7 +254,7 @@ namespace command_line::get_arg(args, opts.webhook_ssl_verification), std::chrono::minutes{command_line::get_arg(args, opts.rates_interval)}, command_line::get_arg(args, opts.scan_threads), - command_line::get_arg(args, opts.create_queue_max), + command_line::get_arg(args, opts.create_queue_max) }; prog.rest_config.threads = std::max(std::size_t(1), prog.rest_config.threads); @@ -273,6 +277,7 @@ namespace MINFO("Using monerod ZMQ RPC at " << ctx.daemon_address()); auto client = lws::scanner::sync(disk.clone(), ctx.connect().value()).value(); + const auto enable_subaddresses = bool(prog.rest_config.max_subaddresses); const auto webhook_verify = prog.rest_config.webhook_verify; lws::rest_server server{ epee::to_span(prog.rest_servers), prog.admin_rest_servers, disk.clone(), std::move(client), std::move(prog.rest_config) @@ -283,7 +288,7 @@ namespace MINFO("Listening for REST admin clients at " << address); // blocks until SIGINT - lws::scanner::run(std::move(disk), std::move(ctx), prog.scan_threads, webhook_verify); + lws::scanner::run(std::move(disk), std::move(ctx), prog.scan_threads, webhook_verify, enable_subaddresses); } } // anonymous diff --git a/src/wire/adapted/array.h b/src/wire/adapted/array.h new file mode 100644 index 0000000..5c98c4f --- /dev/null +++ b/src/wire/adapted/array.h @@ -0,0 +1,96 @@ +// Copyright (c) 2023, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include +#include "span.h" +#include "wire/error.h" +#include "wire/read.h" + +namespace wire +{ + // enable writing of std::array + template + struct is_array> + : std::true_type + {}; + + // `std::array`s of `char` and `uint8_t` are not arrays + template + struct is_array> + : std::false_type + {}; + template + struct is_array> + : std::false_type + {}; + + template + inline void read_bytes(R& source, std::array& dest) + { + source.binary(epee::to_mut_span(dest)); + } + template + inline void read_bytes(R& source, std::array& dest) + { + source.binary(epee::to_mut_span(dest)); + } + + template + inline void write_bytes(W& dest, const std::array& source) + { + source.binary(epee::to_span(source)); + } + template + inline void write_bytes(W& dest, const std::array& source) + { + source.binary(epee::to_span(source)); + } + + // Read a fixed sized array + template + inline void read_bytes(R& source, std::array& dest) + { + std::size_t count = source.start_array(); + const bool json = (count == 0); + if (!json && count != dest.size()) + WIRE_DLOG_THROW(wire::error::schema::array, "Expected array of size " << dest.size()); + + for (auto& elem : dest) + { + if (json && source.is_array_end(count)) + WIRE_DLOG_THROW(wire::error::schema::array, "Expected array of size " << dest.size()); + wire_read::bytes(source, elem); + --count; + } + if (!source.is_array_end(count)) + WIRE_DLOG_THROW(wire::error::schema::array, "Expected array of size " << dest.size()); + source.end_array(); + } +} + diff --git a/tests/unit/db/CMakeLists.txt b/tests/unit/db/CMakeLists.txt index 3c956ec..1d42eca 100644 --- a/tests/unit/db/CMakeLists.txt +++ b/tests/unit/db/CMakeLists.txt @@ -26,7 +26,7 @@ # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -add_library(monero-lws-unit-db OBJECT storage.test.cpp webhook.test.cpp) +add_library(monero-lws-unit-db OBJECT data.test.cpp storage.test.cpp subaddress.test.cpp webhook.test.cpp) target_link_libraries( monero-lws-unit-db monero-lws-unit-framework diff --git a/tests/unit/db/data.test.cpp b/tests/unit/db/data.test.cpp new file mode 100644 index 0000000..e40760f --- /dev/null +++ b/tests/unit/db/data.test.cpp @@ -0,0 +1,76 @@ +// Copyright (c) 2023, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "framework.test.h" + +#include "db/data.h" + +LWS_CASE("db::data::check_subaddress_dict") +{ + EXPECT(lws::db::check_subaddress_dict({lws::db::major_index(0), lws::db::index_ranges{}})); + EXPECT(lws::db::check_subaddress_dict( + { + lws::db::major_index(0), + lws::db::index_ranges{lws::db::index_range{{lws::db::minor_index(0), lws::db::minor_index(0)}}} + } + )); + EXPECT(lws::db::check_subaddress_dict( + { + lws::db::major_index(0), + lws::db::index_ranges{ + lws::db::index_range{{lws::db::minor_index(0), lws::db::minor_index(0)}}, + lws::db::index_range{{lws::db::minor_index(2), lws::db::minor_index(10)}} + } + } + )); + + EXPECT(!lws::db::check_subaddress_dict( + { + lws::db::major_index(0), + lws::db::index_ranges{lws::db::index_range{{lws::db::minor_index(1), lws::db::minor_index(0)}}} + } + )); + EXPECT(!lws::db::check_subaddress_dict( + { + lws::db::major_index(0), + lws::db::index_ranges{ + lws::db::index_range{{lws::db::minor_index(0), lws::db::minor_index(4)}}, + lws::db::index_range{{lws::db::minor_index(1), lws::db::minor_index(10)}} + } + } + )); + EXPECT(!lws::db::check_subaddress_dict( + { + lws::db::major_index(0), + lws::db::index_ranges{ + lws::db::index_range{{lws::db::minor_index(0), lws::db::minor_index(0)}}, + lws::db::index_range{{lws::db::minor_index(1), lws::db::minor_index(10)}} + } + } + )); + +} diff --git a/tests/unit/db/subaddress.test.cpp b/tests/unit/db/subaddress.test.cpp new file mode 100644 index 0000000..9974829 --- /dev/null +++ b/tests/unit/db/subaddress.test.cpp @@ -0,0 +1,422 @@ +// Copyright (c) 2023, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "framework.test.h" + +#include +#include +#include +#include "crypto/crypto.h" // monero/src +#include "db/data.h" +#include "db/storage.h" +#include "db/storage.test.h" +#include "error.h" +#include "wire/error.h" + +namespace +{ + struct user_account + { + lws::db::account_address account; + crypto::secret_key view; + + user_account() + : account{}, view{} + {} + }; + + void check_address_map(lest::env& lest_env, lws::db::storage_reader& reader, const user_account& user, const std::vector& source) + { + SETUP("check_address_map") + { + lws::db::cursor::subaddress_indexes cur = nullptr; + for (const auto& major_entry : source) + { + for (const auto& minor_entry : major_entry.second) + { + for (std::uint64_t elem : boost::counting_range(std::uint64_t(minor_entry[0]), std::uint64_t(minor_entry[1]) + 1)) + { + const lws::db::address_index index{major_entry.first, lws::db::minor_index(elem)}; + auto result = reader.find_subaddress(lws::db::account_id(1), index.get_spend_public(user.account, user.view), cur); + EXPECT(result.has_value()); + EXPECT(result == index); + } + } + } + } + } +} + +LWS_CASE("db::storage::upsert_subaddresses") +{ + user_account user{}; + crypto::generate_keys(user.account.spend_public, user.view); + crypto::generate_keys(user.account.view_public, user.view); + + SETUP("One Account DB") + { + lws::db::test::cleanup_db on_scope_exit{}; + lws::db::storage db = lws::db::test::get_fresh_db(); + const lws::db::block_info last_block = + MONERO_UNWRAP(MONERO_UNWRAP(db.start_read()).get_last_block()); + MONERO_UNWRAP(db.add_account(user.account, user.view)); + + SECTION("Empty get_subaddresses") + { + lws::db::storage_reader reader = MONERO_UNWRAP(db.start_read()); + EXPECT(MONERO_UNWRAP(reader.get_subaddresses(lws::db::account_id(1))).empty()); + } + + SECTION("Upsert Basic") + { + std::vector subs{}; + subs.emplace_back( + lws::db::major_index(0), + lws::db::index_ranges{lws::db::index_range{lws::db::minor_index(1), lws::db::minor_index(100)}} + ); + auto result = db.upsert_subaddresses(lws::db::account_id(1), user.account, user.view, subs, 100); + { + lws::db::storage_reader reader = MONERO_UNWRAP(db.start_read()); + + EXPECT(result.has_value()); + EXPECT(result->size() == 1); + EXPECT(result->at(0).first == lws::db::major_index(0)); + EXPECT(result->at(0).second.size() == 1); + EXPECT(result->at(0).second[0][0] == lws::db::minor_index(1)); + EXPECT(result->at(0).second[0][1] == lws::db::minor_index(100)); + + check_address_map(lest_env, reader, user, subs); + } + subs.back().first = lws::db::major_index(1); + result = db.upsert_subaddresses(lws::db::account_id(1), user.account, user.view, subs, 199); + EXPECT(result.has_error()); + EXPECT(result == lws::error::max_subaddresses); + + lws::db::storage_reader reader = MONERO_UNWRAP(db.start_read()); + const auto fetched = reader.get_subaddresses(lws::db::account_id(1)); + EXPECT(fetched.has_value()); + EXPECT(fetched->size() == 1); + EXPECT(fetched->at(0).first == lws::db::major_index(0)); + EXPECT(fetched->at(0).second.size() == 1); + EXPECT(fetched->at(0).second[0][0] == lws::db::minor_index(1)); + EXPECT(fetched->at(0).second[0][1] == lws::db::minor_index(100)); + } + + SECTION("Upsert Appended") + { + std::vector subs{}; + subs.emplace_back( + lws::db::major_index(0), + lws::db::index_ranges{lws::db::index_range{lws::db::minor_index(1), lws::db::minor_index(100)}} + ); + auto result = db.upsert_subaddresses(lws::db::account_id(1), user.account, user.view, subs, 100); + EXPECT(result.has_value()); + EXPECT(result->size() == 1); + EXPECT(result->at(0).first == lws::db::major_index(0)); + EXPECT(result->at(0).second.size() == 1); + EXPECT(result->at(0).second[0][0] == lws::db::minor_index(1)); + EXPECT(result->at(0).second[0][1] == lws::db::minor_index(100)); + + { + auto reader = MONERO_UNWRAP(db.start_read()); + check_address_map(lest_env, reader, user, subs); + } + + subs.back().second = + lws::db::index_ranges{lws::db::index_range{lws::db::minor_index(101), lws::db::minor_index(200)}}; + result = db.upsert_subaddresses(lws::db::account_id(1), user.account, user.view, subs, 200); + EXPECT(result.has_value()); + EXPECT(result->size() == 1); + EXPECT(result->at(0).first == lws::db::major_index(0)); + EXPECT(result->at(0).second.size() == 1); + EXPECT(result->at(0).second[0][0] == lws::db::minor_index(101)); + EXPECT(result->at(0).second[0][1] == lws::db::minor_index(200)); + + { + auto reader = MONERO_UNWRAP(db.start_read()); + check_address_map(lest_env, reader, user, subs); + } + + subs.back().second = + lws::db::index_ranges{lws::db::index_range{lws::db::minor_index(201), lws::db::minor_index(201)}}; + result = db.upsert_subaddresses(lws::db::account_id(1), user.account, user.view, subs, 200); + EXPECT(result.has_error()); + EXPECT(result == lws::error::max_subaddresses); + + auto reader = MONERO_UNWRAP(db.start_read()); + const auto fetched = reader.get_subaddresses(lws::db::account_id(1)); + EXPECT(fetched.has_value()); + EXPECT(fetched->size() == 1); + EXPECT(fetched->at(0).first == lws::db::major_index(0)); + EXPECT(fetched->at(0).second.size() == 1); + EXPECT(fetched->at(0).second[0][0] == lws::db::minor_index(1)); + EXPECT(fetched->at(0).second[0][1] == lws::db::minor_index(200)); + } + + SECTION("Upsert Prepended") + { + std::vector subs{}; + subs.emplace_back( + lws::db::major_index(0), + lws::db::index_ranges{lws::db::index_range{lws::db::minor_index(101), lws::db::minor_index(200)}} + ); + auto result = db.upsert_subaddresses(lws::db::account_id(1), user.account, user.view, subs, 100); + EXPECT(result.has_value()); + EXPECT(result->size() == 1); + EXPECT(result->at(0).first == lws::db::major_index(0)); + EXPECT(result->at(0).second.size() == 1); + EXPECT(result->at(0).second[0][0] == lws::db::minor_index(101)); + EXPECT(result->at(0).second[0][1] == lws::db::minor_index(200)); + + { + auto reader = MONERO_UNWRAP(db.start_read()); + check_address_map(lest_env, reader, user, subs); + } + + subs.back().second = + lws::db::index_ranges{lws::db::index_range{lws::db::minor_index(1), lws::db::minor_index(100)}}; + + result = db.upsert_subaddresses(lws::db::account_id(1), user.account, user.view, subs, 199); + EXPECT(result.has_error()); + EXPECT(result == lws::error::max_subaddresses); + + result = db.upsert_subaddresses(lws::db::account_id(1), user.account, user.view, subs, 200); + EXPECT(result.has_value()); + EXPECT(result->size() == 1); + EXPECT(result->at(0).first == lws::db::major_index(0)); + EXPECT(result->at(0).second.size() == 1); + EXPECT(result->at(0).second[0][0] == lws::db::minor_index(1)); + EXPECT(result->at(0).second[0][1] == lws::db::minor_index(100)); + + lws::db::storage_reader reader = MONERO_UNWRAP(db.start_read()); + check_address_map(lest_env, reader, user, subs); + + const auto fetched = reader.get_subaddresses(lws::db::account_id(1)); + EXPECT(fetched.has_value()); + EXPECT(fetched->size() == 1); + EXPECT(fetched->at(0).first == lws::db::major_index(0)); + EXPECT(fetched->at(0).second.size() == 1); + EXPECT(fetched->at(0).second[0][0] == lws::db::minor_index(1)); + EXPECT(fetched->at(0).second[0][1] == lws::db::minor_index(200)); + } + + SECTION("Upsert Wrapped") + { + std::vector subs{}; + subs.emplace_back( + lws::db::major_index(0), + lws::db::index_ranges{lws::db::index_range{lws::db::minor_index(101), lws::db::minor_index(200)}} + ); + auto result = db.upsert_subaddresses(lws::db::account_id(1), user.account, user.view, subs, 100); + EXPECT(result.has_value()); + EXPECT(result->size() == 1); + EXPECT(result->at(0).first == lws::db::major_index(0)); + EXPECT(result->at(0).second.size() == 1); + EXPECT(result->at(0).second[0][0] == lws::db::minor_index(101)); + EXPECT(result->at(0).second[0][1] == lws::db::minor_index(200)); + + { + auto reader = MONERO_UNWRAP(db.start_read()); + check_address_map(lest_env, reader, user, subs); + } + + subs.back().second = + lws::db::index_ranges{lws::db::index_range{lws::db::minor_index(1), lws::db::minor_index(300)}}; + + result = db.upsert_subaddresses(lws::db::account_id(1), user.account, user.view, subs, 299); + EXPECT(result.has_error()); + EXPECT(result == lws::error::max_subaddresses); + + result = db.upsert_subaddresses(lws::db::account_id(1), user.account, user.view, subs, 300); + EXPECT(result.has_value()); + EXPECT(result->size() == 1); + EXPECT(result->at(0).first == lws::db::major_index(0)); + EXPECT(result->at(0).second.size() == 2); + EXPECT(result->at(0).second[0][0] == lws::db::minor_index(1)); + EXPECT(result->at(0).second[0][1] == lws::db::minor_index(100)); + EXPECT(result->at(0).second[1][0] == lws::db::minor_index(201)); + EXPECT(result->at(0).second[1][1] == lws::db::minor_index(300)); + + lws::db::storage_reader reader = MONERO_UNWRAP(db.start_read()); + check_address_map(lest_env, reader, user, subs); + const auto fetched = reader.get_subaddresses(lws::db::account_id(1)); + EXPECT(fetched.has_value()); + EXPECT(fetched->size() == 1); + EXPECT(fetched->at(0).first == lws::db::major_index(0)); + EXPECT(fetched->at(0).second.size() == 1); + EXPECT(fetched->at(0).second[0][0] == lws::db::minor_index(1)); + EXPECT(fetched->at(0).second[0][1] == lws::db::minor_index(300)); + } + + SECTION("Upsert After") + { + std::vector subs{}; + subs.emplace_back( + lws::db::major_index(0), + lws::db::index_ranges{lws::db::index_range{lws::db::minor_index(1), lws::db::minor_index(100)}} + ); + auto result = db.upsert_subaddresses(lws::db::account_id(1), user.account, user.view, subs, 100); + EXPECT(result.has_value()); + EXPECT(result->size() == 1); + EXPECT(result->at(0).first == lws::db::major_index(0)); + EXPECT(result->at(0).second.size() == 1); + EXPECT(result->at(0).second[0][0] == lws::db::minor_index(1)); + EXPECT(result->at(0).second[0][1] == lws::db::minor_index(100)); + + { + auto reader = MONERO_UNWRAP(db.start_read()); + check_address_map(lest_env, reader, user, subs); + } + + subs.back().second = + lws::db::index_ranges{lws::db::index_range{lws::db::minor_index(102), lws::db::minor_index(200)}}; + result = db.upsert_subaddresses(lws::db::account_id(1), user.account, user.view, subs, 198); + EXPECT(result.has_error()); + EXPECT(result == lws::error::max_subaddresses); + + result = db.upsert_subaddresses(lws::db::account_id(1), user.account, user.view, subs, 199); + EXPECT(result.has_value()); + EXPECT(result->size() == 1); + EXPECT(result->at(0).first == lws::db::major_index(0)); + EXPECT(result->at(0).second.size() == 1); + EXPECT(result->at(0).second[0][0] == lws::db::minor_index(102)); + EXPECT(result->at(0).second[0][1] == lws::db::minor_index(200)); + + auto reader = MONERO_UNWRAP(db.start_read()); + check_address_map(lest_env, reader, user, subs); + const auto fetched = reader.get_subaddresses(lws::db::account_id(1)); + EXPECT(fetched.has_value()); + EXPECT(fetched->size() == 1); + EXPECT(fetched->at(0).first == lws::db::major_index(0)); + EXPECT(fetched->at(0).second.size() == 2); + EXPECT(fetched->at(0).second[0][0] == lws::db::minor_index(1)); + EXPECT(fetched->at(0).second[0][1] == lws::db::minor_index(100)); + EXPECT(fetched->at(0).second[1][0] == lws::db::minor_index(102)); + EXPECT(fetched->at(0).second[1][1] == lws::db::minor_index(200)); + } + + SECTION("Upsert Before") + { + std::vector subs{}; + subs.emplace_back( + lws::db::major_index(0), + lws::db::index_ranges{lws::db::index_range{lws::db::minor_index(101), lws::db::minor_index(200)}} + ); + auto result = db.upsert_subaddresses(lws::db::account_id(1), user.account, user.view, subs, 100); + EXPECT(result.has_value()); + EXPECT(result->size() == 1); + EXPECT(result->at(0).first == lws::db::major_index(0)); + EXPECT(result->at(0).second.size() == 1); + EXPECT(result->at(0).second[0][0] == lws::db::minor_index(101)); + EXPECT(result->at(0).second[0][1] == lws::db::minor_index(200)); + + { + auto reader = MONERO_UNWRAP(db.start_read()); + check_address_map(lest_env, reader, user, subs); + } + + subs.back().second = + lws::db::index_ranges{lws::db::index_range{lws::db::minor_index(1), lws::db::minor_index(99)}}; + result = db.upsert_subaddresses(lws::db::account_id(1), user.account, user.view, subs, 198); + EXPECT(result.has_error()); + EXPECT(result == lws::error::max_subaddresses); + + result = db.upsert_subaddresses(lws::db::account_id(1), user.account, user.view, subs, 199); + EXPECT(result.has_value()); + EXPECT(result->size() == 1); + EXPECT(result->at(0).first == lws::db::major_index(0)); + EXPECT(result->at(0).second.size() == 1); + EXPECT(result->at(0).second[0][0] == lws::db::minor_index(1)); + EXPECT(result->at(0).second[0][1] == lws::db::minor_index(99)); + + auto reader = MONERO_UNWRAP(db.start_read()); + check_address_map(lest_env, reader, user, subs); + const auto fetched = reader.get_subaddresses(lws::db::account_id(1)); + EXPECT(fetched.has_value()); + EXPECT(fetched->size() == 1); + EXPECT(fetched->at(0).first == lws::db::major_index(0)); + EXPECT(fetched->at(0).second.size() == 2); + EXPECT(fetched->at(0).second[0][0] == lws::db::minor_index(1)); + EXPECT(fetched->at(0).second[0][1] == lws::db::minor_index(99)); + EXPECT(fetched->at(0).second[1][0] == lws::db::minor_index(101)); + EXPECT(fetched->at(0).second[1][1] == lws::db::minor_index(200)); + } + + SECTION("Upsert Encapsulated") + { + std::vector subs{}; + subs.emplace_back( + lws::db::major_index(0), + lws::db::index_ranges{lws::db::index_range{lws::db::minor_index(1), lws::db::minor_index(200)}} + ); + auto result = db.upsert_subaddresses(lws::db::account_id(1), user.account, user.view, subs, 200); + EXPECT(result.has_value()); + EXPECT(result->size() == 1); + EXPECT(result->at(0).first == lws::db::major_index(0)); + EXPECT(result->at(0).second.size() == 1); + EXPECT(result->at(0).second[0][0] == lws::db::minor_index(1)); + EXPECT(result->at(0).second[0][1] == lws::db::minor_index(200)); + + { + auto reader = MONERO_UNWRAP(db.start_read()); + check_address_map(lest_env, reader, user, subs); + } + + subs.back().second = + lws::db::index_ranges{lws::db::index_range{lws::db::minor_index(5), lws::db::minor_index(99)}}; + result = db.upsert_subaddresses(lws::db::account_id(1), user.account, user.view, subs, 300); + EXPECT(result.has_value()); + EXPECT(result->size() == 0); + + auto reader = MONERO_UNWRAP(db.start_read()); + check_address_map(lest_env, reader, user, subs); + const auto fetched = reader.get_subaddresses(lws::db::account_id(1)); + EXPECT(fetched.has_value()); + EXPECT(fetched->size() == 1); + EXPECT(fetched->at(0).first == lws::db::major_index(0)); + EXPECT(fetched->at(0).second.size() == 1); + EXPECT(fetched->at(0).second[0][0] == lws::db::minor_index(1)); + EXPECT(fetched->at(0).second[0][1] == lws::db::minor_index(200)); + } + + + SECTION("Bad subaddress_dict") + { + std::vector subs{}; + subs.emplace_back( + lws::db::major_index(0), + lws::db::index_ranges{lws::db::index_range{lws::db::minor_index(1), lws::db::minor_index(100)}} + ); + subs.back().second.push_back( + lws::db::index_range{lws::db::minor_index(101), lws::db::minor_index(200)} + ); + auto result = db.upsert_subaddresses(lws::db::account_id(1), user.account, user.view, subs, 100); + EXPECT(result.has_error()); + EXPECT(result == wire::error::schema::array); + } + } +}