From 16111cae2cdd5fa7225c0864e273418cad75d885 Mon Sep 17 00:00:00 2001 From: Lee *!* Clagett Date: Thu, 4 Dec 2025 14:29:41 -0500 Subject: [PATCH] Add support for subaddress lookahead (#195) --- src/admin_main.cpp | 7 + src/db/data.cpp | 7 +- src/db/data.h | 17 +- src/db/storage.cpp | 704 +++++++++++++++++++++++++-------- src/db/storage.h | 14 +- src/rest_server.cpp | 95 ++++- src/rest_server.h | 1 + src/rpc/admin.cpp | 3 +- src/rpc/admin.h | 13 +- src/rpc/light_wallet.cpp | 20 +- src/rpc/light_wallet.h | 10 + src/scanner.cpp | 49 ++- src/scanner.h | 2 +- src/server_main.cpp | 9 +- src/wire/wrapper/array.h | 5 + tests/unit/db/print.test.h | 39 ++ tests/unit/db/storage.test.cpp | 201 ++++++++++ tests/unit/lest.hpp | 3 + tests/unit/rest.test.cpp | 178 ++++++++- tests/unit/scanner.test.cpp | 421 +++++++++++++++++++- 20 files changed, 1588 insertions(+), 210 deletions(-) create mode 100644 tests/unit/db/print.test.h diff --git a/src/admin_main.cpp b/src/admin_main.cpp index 163c026..917193c 100644 --- a/src/admin_main.cpp +++ b/src/admin_main.cpp @@ -96,12 +96,14 @@ namespace struct options : lws::options { const command_line::arg_descriptor show_sensitive; + const command_line::arg_descriptor max_subaddresses; const command_line::arg_descriptor command; const command_line::arg_descriptor> arguments; options() : lws::options() , show_sensitive{"show-sensitive", "Show view keys", false} + , max_subaddresses{"max-subaddresses", "Max subaddresses allowed on import/create", 0} , command{"command", "Admin command to execute", ""} , arguments{"arguments", "Arguments to command"} {} @@ -110,6 +112,7 @@ namespace { lws::options::prepare(description); command_line::add_arg(description, show_sensitive); + command_line::add_arg(description, max_subaddresses); command_line::add_arg(description, command); command_line::add_arg(description, arguments); } @@ -119,6 +122,7 @@ namespace { lws::db::storage disk; std::vector arguments; + std::uint32_t max_subaddresses; bool show_sensitive; }; @@ -154,6 +158,7 @@ namespace lws::rpc::address_requests req{ get_addresses(epee::to_span(prog.arguments)), + prog.max_subaddresses, MONERO_UNWRAP(lws::db::request_from_string(prog.arguments[0])) }; run_command(lws::rpc::accept_requests, out, std::move(prog.disk), std::move(req)); @@ -250,6 +255,7 @@ namespace lws::rpc::address_requests req{ get_addresses(epee::to_span(prog.arguments)), + prog.max_subaddresses, lws::db::request_from_string(prog.arguments[0]).value() }; run_command(lws::rpc::reject_requests, out, std::move(prog.disk), std::move(req)); @@ -381,6 +387,7 @@ namespace lws::db::storage::open(command_line::get_arg(args, opts.db_path).c_str(), 0) }; + prog.max_subaddresses = command_line::get_arg(args, opts.max_subaddresses); prog.show_sensitive = command_line::get_arg(args, opts.show_sensitive); auto cmd = args[opts.command.name]; if (cmd.empty()) diff --git a/src/db/data.cpp b/src/db/data.cpp index 6c39207..33cfb0d 100644 --- a/src/db/data.cpp +++ b/src/db/data.cpp @@ -189,7 +189,9 @@ namespace db WIRE_FIELD(start_height), wire::field("creation_time", self.creation), wire::field("admin", admin), - wire::field("generated_locally", generated_locally) + wire::field("generated_locally", generated_locally), + WIRE_FIELD(lookahead), + WIRE_FIELD(lookahead_fail) ); } @@ -403,7 +405,8 @@ namespace db WIRE_FIELD(address), wire::optional_field("view_key", key), WIRE_FIELD(start_height), - wire::field("generated_locally", generated) + wire::field("generated_locally", generated), + WIRE_FIELD(lookahead) ); } diff --git a/src/db/data.h b/src/db/data.h index d637f98..57b0085 100644 --- a/src/db/data.h +++ b/src/db/data.h @@ -134,10 +134,16 @@ namespace db enum class major_index : std::uint32_t { primary = 0 }; WIRE_AS_INTEGER(major_index); + inline constexpr std::uint32_t to_uint(const major_index src) noexcept + { return std::uint32_t(src); } + //! Minor index of a subaddress enum class minor_index : std::uint32_t { primary = 0 }; WIRE_AS_INTEGER(minor_index); + inline constexpr std::uint32_t to_uint(const minor_index src) noexcept + { return std::uint32_t(src); } + //! Range within a major index using index_range = std::array; @@ -185,8 +191,10 @@ namespace db account_time creation; //!< Time account first appeared in database. account_flags flags; //!< Additional account info bitmask. char reserved[3]; + address_index lookahead; + block_id lookahead_fail; }; - static_assert(sizeof(account) == (4 * 2) + 64 + 32 + (8 * 2) + (4 * 2), "padding in account"); + static_assert(sizeof(account) == (4 * 2) + 64 + 32 + (8 * 2) + (4 * 2) + (4 * 2) + 8, "padding in account"); void write_bytes(wire::writer&, const account&, bool show_key = false); //! Used with quick and full sync mode @@ -331,8 +339,9 @@ namespace db account_time creation; //!< Time the request was created. account_flags creation_flags; //!< Generated locally? char reserved[3]; + address_index lookahead; //!< Desired subaddress lookahead }; - static_assert(sizeof(request_info) == 64 + 32 + 8 + (4 * 2), "padding in request_info"); + static_assert(sizeof(request_info) == 64 + 32 + 8 + (4 * 2) + (4*2), "padding in request_info"); void write_bytes(wire::writer& dest, const request_info& self, bool show_key = false); enum class webhook_type : std::uint8_t @@ -428,6 +437,10 @@ namespace db { return left.maj_i == right.maj_i && left.min_i == right.min_i; } + inline constexpr bool operator!=(address_index const& left, address_index const& right) noexcept + { + return left.maj_i != right.maj_i || left.min_i != right.min_i; + } inline constexpr bool operator<(address_index const& left, address_index const& right) noexcept { diff --git a/src/db/storage.cpp b/src/db/storage.cpp index a9c69f3..dde0448 100644 --- a/src/db/storage.cpp +++ b/src/db/storage.cpp @@ -80,7 +80,22 @@ namespace lws namespace db { namespace v0 - { + { + //! Original DB value, no lookahead + struct account + { + account_id id; //!< Must be first for LMDB optimizations + account_time access; //!< Last time `get_address_info` was called. + account_address address; + view_key key; //!< Doubles as authorization handle for REST API. + block_id scan_height; //!< Last block scanned; check-ins are always by block + block_id start_height; //!< Account started scanning at this block height + account_time creation; //!< Time account first appeared in database. + account_flags flags; //!< Additional account info bitmask. + char reserved[3]; + }; + static_assert(sizeof(account) == (4 * 2) + 64 + 32 + (8 * 2) + (4 * 2), "padding in account"); + //! Orignal DB value, with no txn fee struct output { @@ -130,6 +145,17 @@ namespace db crypto::hash payment_id; //!< Unencrypted only, can't decrypt spend }; static_assert(sizeof(spend) == 8 + 32 * 2 + 8 * 4 + 4 + 3 + 1 + 32, "padding in spend"); + + struct request_info + { + account_address address;//!< Must be first for LMDB optimizations + view_key key; + block_id start_height; + account_time creation; //!< Time the request was created. + account_flags creation_flags; //!< Generated locally? + char reserved[3]; + }; + static_assert(sizeof(request_info) == 64 + 32 + 8 + (4 * 2), "padding in request_info"); } namespace v1 @@ -302,8 +328,11 @@ namespace db constexpr const lmdb::basic_table pows{ "pow_by_id", (MDB_CREATE | MDB_DUPSORT), MONERO_SORT_BY(block_pow, id) }; + constexpr const lmdb::basic_table accounts_v0{ + "accounts_by_status,id", MDB_DUPSORT, MONERO_SORT_BY(v0::account, id) + }; constexpr const lmdb::basic_table accounts{ - "accounts_by_status,id", (MDB_CREATE | MDB_DUPSORT), MONERO_SORT_BY(account, id) + "accounts_v1_by_status,id", (MDB_CREATE | MDB_DUPSORT), MONERO_SORT_BY(account, id) }; constexpr const lmdb::basic_table accounts_by_address( "accounts_by_address", (MDB_CREATE | MDB_DUPSORT), MONERO_COMPARE(account_by_address, address.view_public) @@ -329,8 +358,11 @@ namespace db constexpr const lmdb::basic_table images{ "key_images_by_output_id,image", (MDB_CREATE | MDB_DUPSORT), MONERO_COMPARE(db::key_image, value) }; + constexpr const lmdb::basic_table requests_v0{ + "requests_by_type,address", MDB_DUPSORT, MONERO_COMPARE(v0::request_info, address.spend_public) + }; constexpr const lmdb::basic_table requests{ - "requests_by_type,address", (MDB_CREATE | MDB_DUPSORT), MONERO_COMPARE(request_info, address.spend_public) + "requests_v1_by_type,address", (MDB_CREATE | MDB_DUPSORT), MONERO_COMPARE(request_info, address.spend_public) }; constexpr const lmdb::msgpack_table webhooks{ "webhooks_by_account_id,payment_id", (MDB_CREATE | MDB_DUPSORT), &lmdb::less @@ -660,6 +692,14 @@ namespace db } while (err == 0); return log_lmdb_error(err, __LINE__, __FILE__); } + + template + T add_and_clamp(T start, T count) noexcept + { + if (std::numeric_limits::max() - count < start) + return std::numeric_limits::max(); + return start + count; + } } // anonymous struct storage_internal : lws_lmdb::database @@ -721,6 +761,18 @@ namespace db else if (v0_spends != lmdb::error(MDB_NOTFOUND)) MONERO_THROW(v0_spends.error(), "Error opening old spends table"); + const auto v0_accounts = accounts_v0.open(*txn); + if (v0_accounts) + MONERO_UNWRAP(convert_table(*txn, *v0_accounts, tables.accounts)); + else if (v0_accounts != lmdb::error(MDB_NOTFOUND)) + MONERO_THROW(v0_accounts.error(), "Error opening old accounts table"); + + const auto v0_requests = requests_v0.open(*txn); + if (v0_requests) + MONERO_UNWRAP(convert_table(*txn, *v0_accounts, tables.requests)); + else if (v0_requests != lmdb::error(MDB_NOTFOUND)) + MONERO_THROW(v0_requests.error(), "Error open old requests table"); + check_blockchain(*txn, tables.blocks); check_pow(*txn, tables.pows); MONERO_UNWRAP(this->commit(std::move(txn))); @@ -2116,9 +2168,301 @@ namespace db namespace { + expect> do_upsert(MDB_cursor& ranges_cur, MDB_cursor& indexes_cur, const account_id id, const account_address& address, const crypto::secret_key& view_key, std::vector subaddrs, const std::uint32_t max_subaddr) + { + std::size_t subaddr_count = 0; + std::vector out{}; + index_ranges new_dict{}; + const auto add_out = [&out] (major_index major, index_range minor) + { + if (out.empty() || out.back().first != major) + out.emplace_back(major, index_ranges{std::vector{minor}}); + else + out.back().second.get_container().push_back(minor); + }; + + const auto check_max_range = [&subaddr_count, max_subaddr] (const index_range& range) -> bool + { + const auto more = std::uint32_t(range[1]) - std::uint32_t(range[0]); + if (max_subaddr - subaddr_count <= more) + return false; + subaddr_count += more + 1; + return true; + }; + const auto check_max_ranges = [&check_max_range] (const index_ranges& ranges) -> bool + { + for (const auto& range : ranges.get_container()) + { + if (!check_max_range(range)) + return false; + } + return true; + }; + + MDB_val key = lmdb::to_val(id); + MDB_val value{}; + int err = mdb_cursor_get(&indexes_cur, &key, &value, MDB_SET); + if (err) + { + if (err != MDB_NOTFOUND) + return log_lmdb_error(err, __LINE__, __FILE__); + } + else + { + MLWS_LMDB_CHECK(mdb_cursor_count(&indexes_cur, &subaddr_count)); + if (max_subaddr < subaddr_count) + return {error::max_subaddresses}; + } + + for (auto& major_entry : subaddrs) + { + new_dict.get_container().clear(); + if (!check_subaddress_dict(major_entry)) + { + MERROR("Invalid subaddress_dict given to storage::upsert_subaddrs"); + return {wire::error::schema::array}; + } + + value = lmdb::to_val(major_entry.first); + err = mdb_cursor_get(&ranges_cur, &key, &value, MDB_GET_BOTH); + if (err) + { + if (err != MDB_NOTFOUND) + return log_lmdb_error(err, __LINE__, __FILE__); + if (!check_max_ranges(major_entry.second)) + return {error::max_subaddresses}; + out.push_back(major_entry); + new_dict = std::move(major_entry.second); + } + else // merge new minor index ranges with old + { + auto old_dict = subaddress_ranges.get_value(value); + if (!old_dict) + return old_dict.error(); + MLWS_LMDB_CHECK(mdb_cursor_del(&ranges_cur, 0)); // updated at end + + auto& old_range = old_dict->second.get_container(); + const auto& new_range = major_entry.second.get_container(); + + auto old_loc = old_range.begin(); + auto new_loc = new_range.begin(); + for ( ; old_loc != old_range.end() && new_loc != new_range.end(); ) + { + if (std::uint64_t(new_loc->at(1)) + 1 < std::uint32_t(old_loc->at(0))) + { // new has no overlap with existing + if (!check_max_range(*new_loc)) + return {error::max_subaddresses}; + + new_dict.get_container().push_back(*new_loc); + add_out(major_entry.first, *new_loc); + ++new_loc; + } + else if (std::uint64_t(old_loc->at(1)) + 1 < std::uint32_t(new_loc->at(0))) + { // existing has no overlap with new + new_dict.get_container().push_back(*old_loc); + ++old_loc; + } + else if (old_loc->at(0) <= new_loc->at(0) && new_loc->at(1) <= old_loc->at(1)) + { // new is completely within existing + ++new_loc; + } + else // new overlap at beginning, end, or both + { + if (new_loc->at(0) < old_loc->at(0)) + { // overlap at beginning + const index_range new_range{new_loc->at(0), minor_index(std::uint32_t(old_loc->at(0)) - 1)}; + if (!check_max_range(new_range)) + return {error::max_subaddresses}; + add_out(major_entry.first, new_range); + old_loc->at(0) = new_loc->at(0); + } + if (old_loc->at(1) < new_loc->at(1)) + { // overlap at end + const index_range new_range{minor_index(std::uint32_t(old_loc->at(1)) + 1), new_loc->at(1)}; + if (!check_max_range(new_range)) + return {error::max_subaddresses}; + add_out(major_entry.first, new_range); + old_loc->at(1) = new_loc->at(1); + } + ++new_loc; + } + } + + std::copy(old_loc, old_range.end(), std::back_inserter(new_dict.get_container())); + for ( ; new_loc != new_range.end(); ++new_loc) + { + if (!check_max_range(*new_loc)) + return {error::max_subaddresses}; + new_dict.get_container().push_back(*new_loc); + add_out(major_entry.first, *new_loc); + } + } + + for (const auto& new_indexes : new_dict.get_container()) + { + for (std::uint64_t minor : boost::counting_range(std::uint64_t(new_indexes[0]), std::uint64_t(new_indexes[1]) + 1)) + { + subaddress_map new_value{}; + new_value.index = address_index{major_entry.first, minor_index(minor)}; + new_value.subaddress = new_value.index.get_spend_public(address, view_key); + + key = lmdb::to_val(id); + value = lmdb::to_val(new_value); + const int err = mdb_cursor_put(&indexes_cur, &key, &value, MDB_NODUPDATA); + if (err && err != MDB_KEYEXIST) + return log_lmdb_error(err, __LINE__, __FILE__); + } + } + + const expect value_bytes = + subaddress_ranges.make_value(major_entry.first, new_dict); + if (!value_bytes) + return value_bytes.error(); + key = lmdb::to_val(id); + value = MDB_val{value_bytes->size(), const_cast(static_cast(value_bytes->data()))}; + MLWS_LMDB_CHECK(mdb_cursor_put(&ranges_cur, &key, &value, MDB_NODUPDATA)); + } + + return {std::move(out)}; + } + + expect do_lookahead(MDB_cursor& outputs_cur, MDB_cursor& ranges_cur, MDB_cursor& indexes_cur, const account_id id, const account_address& address, const crypto::secret_key& view_key, const address_index& lookahead, const std::uint32_t max_subaddresses) + { + const auto major = to_uint(lookahead.maj_i); + const auto minor = to_uint(lookahead.min_i); + if (!major || !minor) + return success(); + + // Quick fail check + if (std::numeric_limits::max() < major / minor) + return {error::max_subaddresses}; + if (max_subaddresses < major * minor) + return {error::max_subaddresses}; + + MDB_val key = lmdb::to_val(id); + MDB_val value{}; + + std::vector initial{}; + initial.resize(major); + + /* Cycle through all received subaddresses, record highest minor index + within each major index. */ + { + int err = mdb_cursor_get(&outputs_cur, &key, &value, MDB_SET); + if (err && err != MDB_NOTFOUND) + return log_lmdb_error(err, __LINE__, __FILE__); + if (!err) + { + err = mdb_cursor_get(&outputs_cur, &key, &value, MDB_FIRST_DUP); + for (;;) + { + if (err) + { + if (err == MDB_NOTFOUND) + break; + return log_lmdb_error(err, __LINE__, __FILE__); + } + + const auto receipient = outputs.get_value(value); + if (!receipient) + return receipient.error(); + + if (!receipient->is_zero()) + { + const auto index = to_uint(receipient->maj_i); + const auto needed = add_and_clamp(index, major); + const auto this_minor = add_and_clamp(minor - 1, to_uint(receipient->min_i)); + + // Quick Sanity Check before vector expansion + if (std::numeric_limits::max() < needed / minor) + return {error::max_subaddresses}; + if (max_subaddresses < needed * minor) + return {error::max_subaddresses}; + + static_assert( + std::numeric_limits::max() <= + std::numeric_limits::max() + ); + + initial.resize(std::max(initial.size(), std::size_t(needed))); + auto& elem = initial.at(index); + if (!elem.second.get_container().empty()) + { + auto& current = std::get<1>(elem.second.get_container().at(0)); + current = std::max(current, minor_index(this_minor)); + } + else + elem = {major_index(major), index_ranges{{index_range{minor_index(0), minor_index(this_minor)}}}}; + + err = mdb_cursor_get(&outputs_cur, &key, &value, MDB_NEXT_DUP); + } + } + } + } + + // fill in every gap with a default {0, minor} subaddress range + { + const index_ranges default_{{index_range{minor_index(0), minor_index(minor - 1)}}}; + + std::size_t i = -1; + std::uint32_t total_subs = 0; + for (auto& elem : initial) + { + ++i; + + elem.first = major_index(i); + auto this_minor = minor; + if (elem.second.get_container().empty()) + elem.second.get_container() = default_.get_container(); + else + this_minor = to_uint(std::get<1>(elem.second.get_container().at(0))); + + if (max_subaddresses - total_subs < this_minor) + return {error::max_subaddresses}; + total_subs += this_minor; + } + } + + const auto upserted = do_upsert(ranges_cur, indexes_cur, id, address, view_key, std::move(initial), max_subaddresses); + if (!upserted) + return upserted.error(); + return success(); + } + + expect do_lookahead(account& user, MDB_cursor& outputs_cur, MDB_cursor& ranges_cur, MDB_cursor& indexes_cur, const address_index& lookahead, const std::uint32_t max_subaddresses) + { + crypto::secret_key key; + static_assert(sizeof(key) == sizeof(user.key)); + std::memcpy(std::addressof(unwrap(unwrap(key))), std::addressof(user.key), sizeof(key)); + const expect attempt = + do_lookahead(outputs_cur, ranges_cur, indexes_cur, user.id, user.address, key, lookahead, max_subaddresses); + if (attempt == error::max_subaddresses) + { + user.lookahead = lookahead; + user.lookahead_fail = std::max(block_id(1), user.start_height); + return success(); + } + else if (attempt) + user.lookahead = lookahead; + + return attempt; + } + //! \return Success, even if `address` was not found (designed for expect - change_height(MDB_cursor& accounts_cur, MDB_cursor& accounts_ba_cur, MDB_cursor& accounts_bh_cur, MDB_cursor& outputs_cur, MDB_cursor& spends_cur, MDB_cursor& images_cur, block_id height, account_address const& address) + change_height( + MDB_cursor& accounts_cur, + MDB_cursor& accounts_ba_cur, + MDB_cursor& accounts_bh_cur, + MDB_cursor& outputs_cur, + MDB_cursor& spends_cur, + MDB_cursor& images_cur, + MDB_cursor* ranges_cur, + MDB_cursor* indexes_cur, + block_id height, + account_address const& address, + address_index lookahead = {}, + std::uint32_t max_subaddresses = 0) { MDB_val key = lmdb::to_val(by_address_version); MDB_val value = lmdb::to_val(address); @@ -2146,6 +2490,15 @@ namespace db const block_id current_height = user->scan_height; user->scan_height = std::min(height, user->scan_height); user->start_height = std::min(height, user->start_height); + if (height <= user->lookahead_fail) + user->lookahead_fail = block_id(0); + + // rollback before lookahead + MONERO_CHECK(rollback_outputs(user->id, height, outputs_cur)); + MONERO_CHECK(rollback_spends(user->id, height, spends_cur, images_cur)); + + if (!lookahead.is_zero() && ranges_cur && indexes_cur) + MONERO_CHECK(do_lookahead(*user, outputs_cur, *ranges_cur, *indexes_cur, lookahead, max_subaddresses)); key = lmdb::to_val(lookup->status); value = lmdb::to_val(*user); @@ -2165,9 +2518,6 @@ namespace db mdb_cursor_put(&accounts_bh_cur, &key, &value, MDB_NODUPDATA) ); - MONERO_CHECK(rollback_outputs(user->id, height, outputs_cur)); - MONERO_CHECK(rollback_spends(user->id, height, spends_cur, images_cur)); - return success(); } } @@ -2216,7 +2566,7 @@ namespace db for (account_address const& address : addresses) { const expect changed = change_height( - *accounts_cur, *accounts_ba_cur, *accounts_bh_cur, *outputs_cur, *spends_cur, *images_cur, height, address + *accounts_cur, *accounts_ba_cur, *accounts_bh_cur, *outputs_cur, *spends_cur, *images_cur, nullptr, nullptr, height, address ); if (changed) updated.push_back(address); @@ -2227,14 +2577,14 @@ namespace db }); } - expect> storage::creation_request(account_address const& address, crypto::secret_key const& key, account_flags flags) noexcept + expect> storage::creation_request(account_address const& address, crypto::secret_key const& key, account_flags flags, address_index lookahead) noexcept { MONERO_PRECOND(db != nullptr); if (!db->create_queue_max) return {lws::error::create_queue_max}; - return db->try_write([this, &address, &key, flags] (MDB_txn& txn) -> expect> + return db->try_write([this, &address, &key, flags, lookahead] (MDB_txn& txn) -> expect> { const expect current_time = get_account_time(); if (!current_time) @@ -2293,6 +2643,7 @@ namespace db info.creation = *current_time; info.start_height = *height; info.creation_flags = flags; + info.lookahead = lookahead; keyv = lmdb::to_val(req); value = lmdb::to_val(info); @@ -2324,10 +2675,10 @@ namespace db }); } - expect storage::import_request(account_address const& address, block_id height) noexcept + expect storage::import_request(account_address const& address, block_id height, address_index lookahead) noexcept { MONERO_PRECOND(db != nullptr); - return db->try_write([this, &address, height] (MDB_txn& txn) -> expect + return db->try_write([this, &address, height, lookahead] (MDB_txn& txn) -> expect { const expect current_time = get_account_time(); if (!current_time) @@ -2351,6 +2702,7 @@ namespace db request_info info{}; info.address = address; info.start_height = height; + info.lookahead = lookahead; const request req = request::import_scan; key = lmdb::to_val(req); @@ -2366,10 +2718,60 @@ namespace db }); } + expect storage::shrink_lookahead(account_address const& address, const address_index& lookahead) noexcept + { + // shrink is simple because the upsert logic never has to be run + + MONERO_PRECOND(db != nullptr); + return db->try_write([this, &address, &lookahead] (MDB_txn& txn) -> expect + { + cursor::accounts accounts_cur; + cursor::accounts_by_address accounts_ba_cur; + + MONERO_CHECK(check_cursor(txn, this->db->tables.accounts, accounts_cur)); + MONERO_CHECK(check_cursor(txn, this->db->tables.accounts_ba, accounts_ba_cur)); + + MDB_val key = lmdb::to_val(by_address_version); + MDB_val value = lmdb::to_val(address); + const int err = mdb_cursor_get(accounts_ba_cur.get(), &key, &value, MDB_GET_BOTH); + if (err == MDB_NOTFOUND) + return {lws::error::account_not_found}; + if (err) + return log_lmdb_error(err, __LINE__, __FILE__); + + const expect lookup = + accounts_by_address.get_value(value); + if (!lookup) + return lookup.error(); + + key = lmdb::to_val(lookup->status); + value = lmdb::to_val(lookup->id); + MLWS_LMDB_CHECK( + mdb_cursor_get(accounts_cur.get(), &key, &value, MDB_GET_BOTH) + ); + + expect user = accounts.get_value(value); + if (!user) + return user.error(); + + if (user->lookahead_fail != block_id(0) || user->lookahead.maj_i < lookahead.maj_i || user->lookahead.min_i < lookahead.min_i) + return {error::max_subaddresses}; + + user->lookahead = lookahead; + value = lmdb::to_val(*user); + key = lmdb::to_val(lookup->status); + MLWS_LMDB_CHECK( + mdb_cursor_put(accounts_cur.get(), &key, &value, MDB_CURRENT) + ); + + return success(); + }); + } + namespace { expect> - create_accounts(MDB_txn& txn, storage_internal::tables_ const& tables, epee::span addresses) + create_accounts(MDB_txn& txn, storage_internal::tables_ const& tables, epee::span addresses, const std::uint32_t max_subaddresses) { std::vector stored{}; stored.reserve(addresses.size()); @@ -2382,11 +2784,17 @@ namespace db cursor::accounts_by_address accounts_ba_cur; cursor::accounts_by_height accounts_bh_cur; cursor::requests requests_cur; + cursor::subaddress_ranges ranges_cur; + cursor::subaddress_indexes indexes_cur; + cursor::outputs outputs_cur; MONERO_CHECK(check_cursor(txn, tables.accounts, accounts_cur)); MONERO_CHECK(check_cursor(txn, tables.accounts_ba, accounts_ba_cur)); MONERO_CHECK(check_cursor(txn, tables.accounts_bh, accounts_bh_cur)); MONERO_CHECK(check_cursor(txn, tables.requests, requests_cur)); + MONERO_CHECK(check_cursor(txn, tables.subaddress_ranges, ranges_cur)); + MONERO_CHECK(check_cursor(txn, tables.subaddress_indexes, indexes_cur)); + MONERO_CHECK(check_cursor(txn, tables.outputs, outputs_cur)); expect last_id = find_last_id(*accounts_cur); if (!last_id) @@ -2422,6 +2830,8 @@ namespace db user.access = *current_time; user.creation = info->creation; user.flags = info->creation_flags; + if (!info->lookahead.is_zero()) + MONERO_CHECK(do_lookahead(user, *outputs_cur, *ranges_cur, *indexes_cur, info->lookahead, max_subaddresses)); const expect added = do_add_account(*accounts_cur, *accounts_ba_cur, *accounts_bh_cur, user); @@ -2440,7 +2850,7 @@ namespace db } expect> - import_accounts(MDB_txn& txn, storage_internal::tables_ const& tables, epee::span addresses) + import_accounts(MDB_txn& txn, storage_internal::tables_ const& tables, epee::span addresses, const std::uint32_t max_subaddresses) { std::vector updated{}; updated.reserve(addresses.size()); @@ -2452,6 +2862,8 @@ namespace db cursor::outputs outputs_cur; cursor::spends spends_cur; cursor::images images_cur; + cursor::subaddress_ranges ranges_cur; + cursor::subaddress_indexes indexes_cur; MONERO_CHECK(check_cursor(txn, tables.accounts, accounts_cur)); MONERO_CHECK(check_cursor(txn, tables.accounts_ba, accounts_ba_cur)); @@ -2460,6 +2872,8 @@ namespace db MONERO_CHECK(check_cursor(txn, tables.outputs, outputs_cur)); MONERO_CHECK(check_cursor(txn, tables.spends, spends_cur)); MONERO_CHECK(check_cursor(txn, tables.images, images_cur)); + MONERO_CHECK(check_cursor(txn, tables.subaddress_ranges, ranges_cur)); + MONERO_CHECK(check_cursor(txn, tables.subaddress_indexes, indexes_cur)); const request req = request::import_scan; for (account_address const& address : addresses) @@ -2477,9 +2891,13 @@ namespace db MLWS_LMDB_CHECK(mdb_cursor_del(requests_cur.get(), 0)); if (!new_height) return new_height.error(); + const expect lookahead = + requests.get_value(value); + if (!lookahead) + return lookahead.error(); const expect changed = change_height( - *accounts_cur, *accounts_ba_cur, *accounts_bh_cur, *outputs_cur, *spends_cur, *images_cur, *new_height, address + *accounts_cur, *accounts_ba_cur, *accounts_bh_cur, *outputs_cur, *spends_cur, *images_cur, ranges_cur.get(), indexes_cur.get(), *new_height, address, *lookahead, max_subaddresses ); if (changed) updated.push_back(address); @@ -2491,20 +2909,20 @@ namespace db } // anonymous expect> - storage::accept_requests(request req, epee::span addresses) + storage::accept_requests(request req, epee::span addresses, const std::uint32_t max_subaddresses) { if (addresses.empty()) return std::vector{}; MONERO_PRECOND(db != nullptr); - return db->try_write([this, req, addresses] (MDB_txn& txn) -> expect> + return db->try_write([this, req, addresses, max_subaddresses] (MDB_txn& txn) -> expect> { switch (req) { case request::create: - return create_accounts(txn, this->db->tables, addresses); + return create_accounts(txn, this->db->tables, addresses, max_subaddresses); case request::import_scan: - return import_accounts(txn, this->db->tables, addresses); + return import_accounts(txn, this->db->tables, addresses, max_subaddresses); default: break; } @@ -2920,167 +3338,125 @@ namespace db return db->try_write([this, id, &address, &view_key, &subaddrs, max_subaddr] (MDB_txn& txn) -> expect> { - std::size_t subaddr_count = 0; - std::vector out{}; - index_ranges new_dict{}; - const auto add_out = [&out] (major_index major, index_range minor) - { - if (out.empty() || out.back().first != major) - out.emplace_back(major, index_ranges{std::vector{minor}}); - else - out.back().second.get_container().push_back(minor); - }; - - const auto check_max_range = [&subaddr_count, max_subaddr] (const index_range& range) -> bool - { - const auto more = std::uint32_t(range[1]) - std::uint32_t(range[0]); - if (max_subaddr - subaddr_count <= more) - return false; - subaddr_count += more + 1; - return true; - }; - const auto check_max_ranges = [&check_max_range] (const index_ranges& ranges) -> bool - { - for (const auto& range : ranges.get_container()) - { - if (!check_max_range(range)) - return false; - } - return true; - }; - cursor::subaddress_ranges ranges_cur; cursor::subaddress_indexes indexes_cur; MONERO_CHECK(check_cursor(txn, this->db->tables.subaddress_ranges, ranges_cur)); MONERO_CHECK(check_cursor(txn, this->db->tables.subaddress_indexes, indexes_cur)); - MDB_val key = lmdb::to_val(id); - MDB_val value{}; - int err = mdb_cursor_get(indexes_cur.get(), &key, &value, MDB_SET); + return do_upsert(*ranges_cur, *indexes_cur, id, address, view_key, subaddrs, max_subaddr); + }); + } + + expect storage::update_lookahead(const account_address& address, block_id height, address_index match, std::uint32_t max_subaddresses) + { + MONERO_PRECOND(db != nullptr); + return db->try_write([this, &address, height, match, max_subaddresses] (MDB_txn& txn) -> expect + { + cursor::accounts accounts_cur; + cursor::accounts_by_address accounts_ba_cur; + cursor::subaddress_ranges ranges_cur; + cursor::subaddress_indexes indexes_cur; + + MONERO_CHECK(check_cursor(txn, this->db->tables.accounts, accounts_cur)); + MONERO_CHECK(check_cursor(txn, this->db->tables.accounts_ba, accounts_ba_cur)); + MONERO_CHECK(check_cursor(txn, this->db->tables.subaddress_ranges, ranges_cur)); + MONERO_CHECK(check_cursor(txn, this->db->tables.subaddress_indexes, indexes_cur)); + + MDB_val key = lmdb::to_val(by_address_version); + MDB_val value = lmdb::to_val(address); + const int err = mdb_cursor_get(accounts_ba_cur.get(), &key, &value, MDB_GET_BOTH); + if (err == MDB_NOTFOUND) + return {lws::error::account_not_found}; if (err) + return log_lmdb_error(err, __LINE__, __FILE__); + + const expect lookup = + accounts_by_address.get_value(value); + if (!lookup) + return lookup.error(); + + key = lmdb::to_val(lookup->status); + value = lmdb::to_val(lookup->id); + MLWS_LMDB_CHECK( + mdb_cursor_get(accounts_cur.get(), &key, &value, MDB_GET_BOTH) + ); + + expect user = accounts.get_value(value); + if (!user) + return user.error(); + + const auto major = to_uint(user->lookahead.maj_i); + const auto minor = to_uint(user->lookahead.min_i); + + if (!major || !minor) + return 0; + + expect> upserted{error::max_subaddresses}; + if (major / minor <= std::numeric_limits::max()) { - if (err != MDB_NOTFOUND) - return log_lmdb_error(err, __LINE__, __FILE__); - } - else - { - MLWS_LMDB_CHECK(mdb_cursor_count(indexes_cur.get(), &subaddr_count)); - if (max_subaddr < subaddr_count) - return {error::max_subaddresses}; - } - - for (auto& major_entry : subaddrs) - { - new_dict.get_container().clear(); - if (!check_subaddress_dict(major_entry)) - { - MERROR("Invalid subaddress_dict given to storage::upsert_subaddrs"); - return {wire::error::schema::array}; - } - - value = lmdb::to_val(major_entry.first); - err = mdb_cursor_get(ranges_cur.get(), &key, &value, MDB_GET_BOTH); - if (err) - { - if (err != MDB_NOTFOUND) - return log_lmdb_error(err, __LINE__, __FILE__); - if (!check_max_ranges(major_entry.second)) - return {error::max_subaddresses}; - out.push_back(major_entry); - new_dict = std::move(major_entry.second); - } - else // merge new minor index ranges with old - { - auto old_dict = subaddress_ranges.get_value(value); - if (!old_dict) - return old_dict.error(); - mdb_cursor_del(ranges_cur.get(), 0); // updated at end - - auto& old_range = old_dict->second.get_container(); - const auto& new_range = major_entry.second.get_container(); - - auto old_loc = old_range.begin(); - auto new_loc = new_range.begin(); - for ( ; old_loc != old_range.end() && new_loc != new_range.end(); ) + if (major * minor <= max_subaddresses) + { + std::vector upsertions; + const auto fresh = to_uint(match.maj_i); + const auto end = add_and_clamp(fresh, major); + for (std::uint64_t i = 0; i < end; ++i) { - if (std::uint64_t(new_loc->at(1)) + 1 < std::uint32_t(old_loc->at(0))) - { // new has no overlap with existing - if (!check_max_range(*new_loc)) - return {error::max_subaddresses}; - - new_dict.get_container().push_back(*new_loc); - add_out(major_entry.first, *new_loc); - ++new_loc; - } - else if (std::uint64_t(old_loc->at(1)) + 1 < std::uint32_t(new_loc->at(0))) - { // existing has no overlap with new - new_dict.get_container().push_back(*old_loc); - ++old_loc; - } - else if (old_loc->at(0) <= new_loc->at(0) && new_loc->at(1) <= old_loc->at(1)) - { // new is completely within existing - ++new_loc; - } - else // new overlap at beginning, end, or both + if (i == fresh) { - if (new_loc->at(0) < old_loc->at(0)) - { // overlap at beginning - const index_range new_range{new_loc->at(0), minor_index(std::uint32_t(old_loc->at(0)) - 1)}; - if (!check_max_range(new_range)) - return {error::max_subaddresses}; - add_out(major_entry.first, new_range); - old_loc->at(0) = new_loc->at(0); - } - if (old_loc->at(1) < new_loc->at(1)) - { // overlap at end - const index_range new_range{minor_index(std::uint32_t(old_loc->at(1)) + 1), new_loc->at(1)}; - if (!check_max_range(new_range)) - return {error::max_subaddresses}; - add_out(major_entry.first, new_range); - old_loc->at(1) = new_loc->at(1); - } - ++new_loc; + upsertions.emplace_back( + db::major_index(i), + db::index_ranges{{ + db::index_range{ + db::minor_index(0), + db::minor_index(add_and_clamp(to_uint(match.min_i), minor - 1)) + } + }} + ); + } + else + { + upsertions.emplace_back( + db::major_index(i), + db::index_ranges{{ + db::index_range{db::minor_index(0), db::minor_index(minor - 1)} + }} + ); } } - std::copy(old_loc, old_range.end(), std::back_inserter(new_dict.get_container())); - for ( ; new_loc != new_range.end(); ++new_loc) - { - if (!check_max_range(*new_loc)) - return {error::max_subaddresses}; - new_dict.get_container().push_back(*new_loc); - add_out(major_entry.first, *new_loc); - } + crypto::secret_key viewkey; + static_assert(sizeof(viewkey) == sizeof(user->key)); + std::memcpy(std::addressof(unwrap(unwrap(viewkey))), std::addressof(user->key), sizeof(viewkey)); + upserted = + do_upsert(*ranges_cur, *indexes_cur, user->id, address, viewkey, std::move(upsertions), max_subaddresses); } - - for (const auto& new_indexes : new_dict.get_container()) - { - for (std::uint64_t minor : boost::counting_range(std::uint64_t(new_indexes[0]), std::uint64_t(new_indexes[1]) + 1)) - { - subaddress_map new_value{}; - new_value.index = address_index{major_entry.first, minor_index(minor)}; - new_value.subaddress = new_value.index.get_spend_public(address, view_key); - - key = lmdb::to_val(id); - value = lmdb::to_val(new_value); - - const int err = mdb_cursor_put(indexes_cur.get(), &key, &value, MDB_NODUPDATA); - if (err && err != MDB_KEYEXIST) - return log_lmdb_error(err, __LINE__, __FILE__); - } - } - - const expect value_bytes = - subaddress_ranges.make_value(major_entry.first, new_dict); - if (!value_bytes) - return value_bytes.error(); - key = lmdb::to_val(id); - value = MDB_val{value_bytes->size(), const_cast(static_cast(value_bytes->data()))}; - MLWS_LMDB_CHECK(mdb_cursor_put(ranges_cur.get(), &key, &value, MDB_NODUPDATA)); } - return {std::move(out)}; + if (!upserted) + { + if (upserted != error::max_subaddresses) + return upserted.error(); + + if (user->lookahead_fail == block_id(0)) + user->lookahead_fail = std::max(block_id(1), height); + else + user->lookahead_fail = std::max(block_id(1), std::min(user->lookahead_fail, height)); + + key = lmdb::to_val(lookup->status); + value = lmdb::to_val(*user); + MLWS_LMDB_CHECK( + mdb_cursor_put(accounts_cur.get(), &key, &value, MDB_CURRENT) + ); + + return -1; + } + + static_assert( + std::numeric_limits::max() <= + std::numeric_limits::max() + ); + return std::min(std::size_t(std::numeric_limits::max()), upserted->size()); }); } diff --git a/src/db/storage.h b/src/db/storage.h index ef43a7f..08c1d07 100644 --- a/src/db/storage.h +++ b/src/db/storage.h @@ -255,17 +255,20 @@ namespace db rescan(block_id height, epee::span addresses); //! Add an account for later approval. For use with the login endpoint. - expect> creation_request(account_address const& address, crypto::secret_key const& key, account_flags flags) noexcept; + expect> creation_request(account_address const& address, crypto::secret_key const& key, account_flags flags, address_index lookahead) noexcept; /*! Request lock height of an existing account. No effect if the `start_height` is already older. */ - expect import_request(account_address const& address, block_id height) noexcept; + expect import_request(account_address const& address, block_id height, address_index lookahead) noexcept; + + //! Shrink but not expand lookahead for `address`. + expect shrink_lookahead(account_address const& address, const address_index& lookahead) noexcept; //! Accept requests by `addresses` of type `req`. \return Accepted addresses. expect> - accept_requests(request req, epee::span addresses); + accept_requests(request req, epee::span addresses, std::uint32_t max_subaddresses); //! Reject requests by `addresses` of type `req`. \return Rejected addresses. expect> @@ -311,6 +314,11 @@ namespace db expect> upsert_subaddresses(account_id id, const account_address& address, const crypto::secret_key& view_key, std::vector subaddrs, std::uint32_t max_subaddresses); + /*! Update lookahead where `match` was a matching subaddress on-chain. + \return The number of new subaddresses added via lookahead, or -1 if + `max_subaddresses was reached. */ + expect update_lookahead(const account_address& address, block_id height, address_index match, std::uint32_t max_subaddresses); + /*! Add webhook to be tracked in the database. The webhook will "call" the specified URL with JSON/msgpack information when the event occurs. diff --git a/src/rest_server.cpp b/src/rest_server.cpp index f27f9b2..4a6d0f4 100644 --- a/src/rest_server.cpp +++ b/src/rest_server.cpp @@ -97,6 +97,7 @@ namespace lws const epee::net_utils::ssl_verification_t webhook_verify; const bool disable_admin_auth; const bool auto_accept_creation; + const bool auto_accept_import; }; struct rest_server_data @@ -265,6 +266,19 @@ namespace lws return {std::make_pair(user->second, std::move(*reader))}; } + bool check_lookahead(connection_data& data, const db::address_index lookahead) + { + const auto minor = to_uint(lookahead.min_i); + if (minor) + { + const auto major = to_uint(lookahead.maj_i); + if (std::numeric_limits::max() < major / minor) + return false; + return major * minor <= data.global->options.max_subaddresses; + } + return true; + } + //! For endpoints that _sometimes_ generate async responses expect async_ready() noexcept { return epee::byte_slice{}; } @@ -521,6 +535,8 @@ namespace lws resp.scanned_height = std::uint64_t(user->first.scan_height); resp.scanned_block_height = resp.scanned_height; resp.start_height = std::uint64_t(user->first.start_height); + resp.lookahead_fail = to_uint(user->first.lookahead_fail); + resp.lookahead = user->first.lookahead; std::vector metas{}; metas.reserve(outputs->count()); @@ -595,6 +611,8 @@ namespace lws resp.start_height = std::uint64_t(user->first.start_height); resp.blockchain_height = std::uint64_t(last->id); resp.transaction_height = resp.blockchain_height; + resp.lookahead_fail = to_uint(user->first.lookahead_fail); + resp.lookahead = user->first.lookahead; // merge input and output info into a single set of txes. @@ -1115,6 +1133,7 @@ namespace lws per_byte_fee, rpc->fee_mask, rpc::safe_uint64(received), + to_uint(user->first.lookahead_fail), std::move(unspent), rpc->fees, std::move(req.creds.key) @@ -1340,13 +1359,28 @@ namespace lws { bool new_request = false; bool fulfilled = false; + db::address_index lookahead{}; { auto user = open_account(req.creds, data.global->disk.clone()); if (!user) return user.error(); data.passed_login = true; - if (user->first.start_height <= db::block_id(req.from_height)) + if (!check_lookahead(data, req.lookahead)) + return {lws::error::max_subaddresses}; + + const auto expanded_depth = [&req] (const auto& record) + { return db::block_id(req.from_height) < record.start_height; }; + + const auto change_lookahead = [&req] (const auto& record) + { + return record.lookahead.maj_i != req.lookahead.maj_i || + record.lookahead.min_i != req.lookahead.min_i; + }; + + lookahead = user->first.lookahead; + const bool lookahead_fail = user->first.lookahead_fail != db::block_id(0); + if (!expanded_depth(user->first) && !change_lookahead(user->first) && !lookahead_fail) fulfilled = true; else { @@ -1358,17 +1392,48 @@ namespace lws if (info != lmdb::error(MDB_NOTFOUND)) return info.error(); - new_request = true; + // Shrink immediately if possible + if (!lookahead_fail && req.lookahead.maj_i <= user->first.lookahead.maj_i && req.lookahead.min_i <= user->first.lookahead.min_i) + { + fulfilled = !expanded_depth(user->first); + new_request = !fulfilled; + // if not same + if (user->first.lookahead.maj_i != req.lookahead.maj_i && user->first.lookahead.min_i != req.lookahead.min_i) + { + MONERO_CHECK(data.global->disk.clone().shrink_lookahead(req.creds.address, req.lookahead)); + lookahead = req.lookahead; + } + } + else + new_request = true; } } } // close reader if (new_request) - MONERO_CHECK(data.global->disk.clone().import_request(req.creds.address, db::block_id(req.from_height))); + { + auto disk = data.global->disk.clone(); + MONERO_CHECK(disk.import_request(req.creds.address, db::block_id(req.from_height), req.lookahead)); + + if (data.global->options.auto_accept_import) + { + const auto accepted = disk.accept_requests(db::request::import_scan, {std::addressof(req.creds.address), 1}, data.global->options.max_subaddresses); + if (!accepted) + { + MERROR("Failed to import account " << db::address_string(req.creds.address) << ": " << accepted.error()); + lookahead = {}; + } + else + { + lookahead = req.lookahead; + fulfilled = true; + } + } + } const char* status = new_request ? "Accepted, waiting for approval" : (fulfilled ? "Approved" : "Waiting for Approval"); - return response{rpc::safe_uint64(0), status, new_request, fulfilled}; + return response{rpc::safe_uint64(0), status, lookahead, new_request, fulfilled}; } }; @@ -1398,24 +1463,33 @@ namespace lws // Do not count a request for account creation as login data.passed_login = true; - return response{false, bool(account->second.flags & db::account_generated_locally)}; + return response{false, bool(account->second.flags & db::account_generated_locally), account->second.lookahead}; } else if (!req.create_account || account != lws::error::account_not_found) return account.error(); } + if (!check_lookahead(data, req.lookahead)) + return {lws::error::max_subaddresses}; + const auto flags = req.generated_locally ? db::account_generated_locally : db::default_account; - const auto hooks = disk.creation_request(req.creds.address, req.creds.key, flags); + const auto hooks = disk.creation_request(req.creds.address, req.creds.key, flags, req.lookahead); if (!hooks) return hooks.error(); if (data.global->options.auto_accept_creation) { - data.passed_login = true; - const auto accepted = disk.accept_requests(db::request::create, {std::addressof(req.creds.address), 1}); + const auto accepted = disk.accept_requests(db::request::create, {std::addressof(req.creds.address), 1}, data.global->options.max_subaddresses); if (!accepted) + { MERROR("Failed to move account " << db::address_string(req.creds.address) << " to available state: " << accepted.error()); + req.lookahead = {}; + } + else + data.passed_login = true; } + else + req.lookahead = {}; if (!hooks->empty()) { @@ -1427,7 +1501,7 @@ namespace lws ); } - return response{true, req.generated_locally}; + return response{true, req.generated_locally, req.lookahead}; } }; @@ -1788,6 +1862,7 @@ namespace lws return error; } + rpc::add_values(req.params, data.global->options); // add max_subaddresses db::storage disk = data.global->disk.clone(); if (!data.global->options.disable_admin_auth) { @@ -2307,7 +2382,7 @@ namespace lws } rest_server::rest_server(epee::span addresses, std::vector admin, db::storage disk, rpc::client client, configuration config) - : global_(std::make_unique(std::move(disk), std::move(client), runtime_options{config.max_subaddresses, config.webhook_verify, config.disable_admin_auth, config.auto_accept_creation})), + : global_(std::make_unique(std::move(disk), std::move(client), runtime_options{config.max_subaddresses, config.webhook_verify, config.disable_admin_auth, config.auto_accept_creation, config.auto_accept_import})), ports_(), workers_() { diff --git a/src/rest_server.h b/src/rest_server.h index 7452147..6b7c6b6 100644 --- a/src/rest_server.h +++ b/src/rest_server.h @@ -66,6 +66,7 @@ namespace lws bool allow_external; bool disable_admin_auth; bool auto_accept_creation; + bool auto_accept_import; }; explicit rest_server(epee::span addresses, std::vector admin, db::storage disk, rpc::client client, configuration config); diff --git a/src/rpc/admin.cpp b/src/rpc/admin.cpp index 925bf85..6f018bc 100644 --- a/src/rpc/admin.cpp +++ b/src/rpc/admin.cpp @@ -164,6 +164,7 @@ namespace lws { namespace rpc void read_bytes(wire::reader& source, address_requests& self) { read_addresses(source, self, WIRE_FIELD(type)); + self.max_subaddresses = 0; } void read_bytes(wire::reader& source, modify_account_req& self) { @@ -205,7 +206,7 @@ namespace lws { namespace rpc expect accept_requests_::operator()(wire::writer& dest, db::storage disk, const request& req) const { - return write_addresses(dest, disk.accept_requests(req.type, epee::to_span(req.addresses))); + return write_addresses(dest, disk.accept_requests(req.type, epee::to_span(req.addresses), req.max_subaddresses)); } expect add_account_::operator()(wire::writer& out, db::storage disk, const request& req) const diff --git a/src/rpc/admin.h b/src/rpc/admin.h index 7344387..63e4fa7 100644 --- a/src/rpc/admin.h +++ b/src/rpc/admin.h @@ -40,7 +40,7 @@ namespace lws { namespace rpc -{ +{ struct add_account_req { db::account_address address; @@ -52,6 +52,7 @@ namespace rpc struct address_requests { std::vector addresses; + std::uint32_t max_subaddresses; db::request type; }; void read_bytes(wire::reader&, address_requests&); @@ -102,6 +103,16 @@ namespace rpc void read_bytes(wire::reader&, webhook_delete_uuid_req&); + // Hack for passing max_subaddresses via cli option + template + inline void add_values(const T&, const U&) noexcept + {} + + template + inline void add_values(address_requests& out, const T& src) noexcept + { out.max_subaddresses = src.max_subaddresses; } + + struct accept_requests_ { using request = address_requests; diff --git a/src/rpc/light_wallet.cpp b/src/rpc/light_wallet.cpp index 3ab20c9..179235a 100644 --- a/src/rpc/light_wallet.cpp +++ b/src/rpc/light_wallet.cpp @@ -266,8 +266,10 @@ namespace lws WIRE_FIELD_COPY(start_height), WIRE_FIELD_COPY(transaction_height), WIRE_FIELD_COPY(blockchain_height), + WIRE_FIELD_DEFAULTED(lookahead_fail, unsigned(0)), WIRE_FIELD(spent_outputs), - WIRE_OPTIONAL_FIELD(rates) + WIRE_OPTIONAL_FIELD(rates), + WIRE_FIELD_DEFAULTED(lookahead, db::address_index{}) ); } @@ -318,6 +320,8 @@ namespace lws WIRE_FIELD_COPY(start_height), WIRE_FIELD_COPY(transaction_height), WIRE_FIELD_COPY(blockchain_height), + WIRE_FIELD_DEFAULTED(lookahead_fail, unsigned(0)), + WIRE_FIELD_DEFAULTED(lookahead, db::address_index{}), wire::optional_field("transactions", wire::array(boost::adaptors::index(self.transactions))) ); } @@ -359,6 +363,7 @@ namespace lws WIRE_FIELD_COPY(per_byte_fee), WIRE_FIELD_COPY(fee_mask), WIRE_FIELD_COPY(amount), + WIRE_FIELD_DEFAULTED(lookahead_fail, unsigned(0)), wire::optional_field("outputs", wire::array(boost::adaptors::transform(self.outputs, expand))), WIRE_FIELD(fees) ); @@ -400,7 +405,8 @@ namespace lws wire::object(source, wire::field("address", std::ref(address)), wire::field("view_key", std::ref(unwrap(unwrap(self.creds.key)))), - WIRE_FIELD_DEFAULTED(from_height, unsigned(0)) + WIRE_FIELD_DEFAULTED(from_height, unsigned(0)), + WIRE_FIELD_DEFAULTED(lookahead, db::address_index{}) ); convert_address(address, self.creds.address); } @@ -411,7 +417,8 @@ namespace lws WIRE_FIELD_COPY(import_fee), WIRE_FIELD(status), WIRE_FIELD_COPY(new_request), - WIRE_FIELD_COPY(request_fulfilled) + WIRE_FIELD_COPY(request_fulfilled), + WIRE_FIELD_COPY(lookahead) ); } @@ -421,6 +428,7 @@ namespace lws wire::object(source, wire::field("address", std::ref(address)), wire::field("view_key", std::ref(unwrap(unwrap(self.creds.key)))), + WIRE_FIELD_DEFAULTED(lookahead, db::address_index{}), WIRE_FIELD(create_account), WIRE_FIELD(generated_locally) ); @@ -428,7 +436,11 @@ namespace lws } void rpc::write_bytes(wire::json_writer& dest, const login_response self) { - wire::object(dest, WIRE_FIELD_COPY(new_address), WIRE_FIELD_COPY(generated_locally)); + wire::object(dest, + WIRE_FIELD_COPY(new_address), + WIRE_FIELD_COPY(generated_locally), + WIRE_FIELD_COPY(lookahead) + ); } void rpc::read_bytes(wire::json_reader& source, provision_subaddrs_request& self) diff --git a/src/rpc/light_wallet.h b/src/rpc/light_wallet.h index 5d6ffb8..7fe89ac 100644 --- a/src/rpc/light_wallet.h +++ b/src/rpc/light_wallet.h @@ -136,6 +136,7 @@ namespace rpc start_height(0), transaction_height(0), blockchain_height(0), + lookahead_fail(0), spent_outputs(), rates(common_error::kInvalidArgument) {} @@ -148,8 +149,10 @@ namespace rpc std::uint64_t start_height; std::uint64_t transaction_height; std::uint64_t blockchain_height; + std::uint64_t lookahead_fail; std::vector spent_outputs; expect rates; + db::address_index lookahead; }; void write_bytes(wire::json_writer&, const get_address_info_response&); @@ -171,7 +174,9 @@ namespace rpc std::uint64_t start_height; std::uint64_t transaction_height; std::uint64_t blockchain_height; + std::uint64_t lookahead_fail; std::vector transactions; + db::address_index lookahead; }; void write_bytes(wire::json_writer&, const get_address_txs_response&); @@ -209,6 +214,7 @@ namespace rpc std::uint64_t per_byte_fee; std::uint64_t fee_mask; safe_uint64 amount; + std::uint64_t lookahead_fail; std::vector>> outputs; std::vector fees; crypto::secret_key user_key; @@ -258,6 +264,7 @@ namespace rpc import_request() = delete; account_credentials creds; std::uint64_t from_height; + db::address_index lookahead; }; void read_bytes(wire::json_reader&, import_request&); @@ -266,6 +273,7 @@ namespace rpc import_response() = delete; safe_uint64 import_fee; std::string status; + db::address_index lookahead; bool new_request; bool request_fulfilled; }; @@ -276,6 +284,7 @@ namespace rpc { login_request() = delete; account_credentials creds; + db::address_index lookahead; bool create_account; bool generated_locally; }; @@ -286,6 +295,7 @@ namespace rpc login_response() = delete; bool new_address; bool generated_locally; + db::address_index lookahead; }; void write_bytes(wire::json_writer&, login_response); diff --git a/src/scanner.cpp b/src/scanner.cpp index a25e85f..9a77a4a 100644 --- a/src/scanner.cpp +++ b/src/scanner.cpp @@ -290,20 +290,47 @@ namespace lws struct subaddress_reader { expect reader; + std::optional disk; db::cursor::subaddress_indexes cur; + const std::uint32_t max_subaddresses; - subaddress_reader(std::optional const& disk, const bool enable_subaddresses) - : reader(common_error::kInvalidArgument), cur(nullptr) + subaddress_reader(std::optional const& disk_in, const std::uint32_t max_subaddresses) + : reader(common_error::kInvalidArgument), disk(), cur(nullptr), max_subaddresses(max_subaddresses) { - if (disk && enable_subaddresses) - { + if (disk_in) + disk = disk_in->clone(); + + if (max_subaddresses) + update_reader(); + } + + void update_reader() + { + if (disk) reader = disk->start_read(); - if (!reader) - MERROR("Subadress lookup failure: " << reader.error().message()); - } + if (!reader) + MERROR("Subadress lookup failure: " << reader.error().message()); } }; + void update_lookahead(const account& user, subaddress_reader& reader, const db::address_index& match, const db::block_id height) + { + if (!reader.disk) + throw std::runtime_error{"Bad DB handle in scanner"}; + + auto upserted = reader.disk->update_lookahead(user.db_address(), height, match, reader.max_subaddresses); + if (upserted) + { + if (0 < *upserted) + reader.update_reader(); // update reader after upsert added new addresses + else if (*upserted < 0) + upserted = {error::max_subaddresses}; + } + + if (!upserted) + MWARNING("Failed to update lookahead for " << user.address() << ": " << upserted.error()); + } + void scan_transaction_base( epee::span users, const db::block_id height, @@ -470,6 +497,8 @@ namespace lws MERROR("Failure when doing subaddress search: " << match.error().message()); continue; // to next available active_derived } + + update_lookahead(user, reader, *match, height); found_pub = true; account_index = *match; break; // additional_derivations loop @@ -578,7 +607,7 @@ namespace lws const auto time = boost::numeric_cast(std::chrono::system_clock::to_time_t(std::chrono::system_clock::now())); - subaddress_reader reader{std::optional{disk.clone()}, opts.enable_subaddresses}; + subaddress_reader reader{std::optional{disk.clone()}, opts.max_subaddresses}; send_webhook sender{disk, client, self}; for (const auto& tx : parsed->txes) scan_transaction_base(users, db::block_id::txpool, time, crypto::hash{}, tx, fake_outs, reader, null_spend{}, sender); @@ -829,7 +858,7 @@ namespace lws ); } - subaddress_reader reader{disk, opts.enable_subaddresses}; + subaddress_reader reader{disk, opts.max_subaddresses}; db::block_difficulty::unsigned_int diff{}; const db::block_id initial_height = db::block_id(fetched->start_height); for (auto block_data : boost::combine(blocks, indices)) @@ -1319,7 +1348,7 @@ namespace lws { if (has_shutdown()) MONERO_THROW(common_error::kInvalidArgument, "this has shutdown"); - if (!lws_server_addr.empty() && (opts.enable_subaddresses || opts.untrusted_daemon)) + if (!lws_server_addr.empty() && (opts.max_subaddresses || opts.untrusted_daemon)) MONERO_THROW(error::configuration, "Cannot use remote scanner with subaddresses or untrusted daemon"); if (lws_server_addr.empty()) diff --git a/src/scanner.h b/src/scanner.h index fd99614..0af8de2 100644 --- a/src/scanner.h +++ b/src/scanner.h @@ -45,7 +45,7 @@ namespace lws { struct scanner_options { - bool enable_subaddresses; + std::uint32_t max_subaddresses; bool untrusted_daemon; bool regtest; }; diff --git a/src/server_main.cpp b/src/server_main.cpp index a4be0d7..43913b5 100644 --- a/src/server_main.cpp +++ b/src/server_main.cpp @@ -86,6 +86,7 @@ namespace const command_line::arg_descriptor untrusted_daemon; const command_line::arg_descriptor regtest; const command_line::arg_descriptor version; + const command_line::arg_descriptor auto_accept_import; static std::string get_default_zmq() { @@ -135,6 +136,7 @@ namespace , untrusted_daemon{"untrusted-daemon", "Perform (expensive) chain-verification and PoW checks", false} , regtest{"regtest", "Run in a regression testing mode", false} , version{"version", "Display version and quit", false} + , auto_accept_import{"auto-accept-import", "Account import requests are automatically accepted", false} {} void prepare(boost::program_options::options_description& description) const @@ -172,6 +174,7 @@ namespace command_line::add_arg(description, untrusted_daemon); command_line::add_arg(description, regtest); command_line::add_arg(description, version); + command_line::add_arg(description, auto_accept_import); } }; @@ -274,7 +277,8 @@ namespace webhook_verify, command_line::get_arg(args, opts.external_bind), command_line::get_arg(args, opts.disable_admin_auth), - command_line::get_arg(args, opts.auto_accept_creation) + command_line::get_arg(args, opts.auto_accept_creation), + command_line::get_arg(args, opts.auto_accept_import) }, command_line::get_arg(args, opts.daemon_rpc), command_line::get_arg(args, opts.daemon_sub), @@ -332,7 +336,6 @@ namespace auto client = scanner.sync(ctx.connect().value(), prog.untrusted_daemon).value(); - const auto enable_subaddresses = bool(prog.rest_config.max_subaddresses); lws::rest_server server{ epee::to_span(prog.rest_servers), prog.admin_rest_servers, std::move(disk), std::move(client), std::move(prog.rest_config) }; @@ -347,7 +350,7 @@ namespace prog.scan_threads, std::move(prog.lws_server_addr), std::move(prog.lws_server_pass), - lws::scanner_options{enable_subaddresses, prog.untrusted_daemon, prog.regtest} + lws::scanner_options{prog.rest_config.max_subaddresses, prog.untrusted_daemon, prog.regtest} ); } } // anonymous diff --git a/src/wire/wrapper/array.h b/src/wire/wrapper/array.h index 99513fb..eba0c4e 100644 --- a/src/wire/wrapper/array.h +++ b/src/wire/wrapper/array.h @@ -152,6 +152,11 @@ namespace wire return {std::move(value)}; } + + template + inline bool operator==(const array_& lhs, const array_& rhs) + { return lhs.get_container() == rhs.get_container(); } + /* Do not register with `is_optional_on_empty` trait, this allows selection on whether an array is mandatory on wire. */ diff --git a/tests/unit/db/print.test.h b/tests/unit/db/print.test.h new file mode 100644 index 0000000..625cd7e --- /dev/null +++ b/tests/unit/db/print.test.h @@ -0,0 +1,39 @@ +// Copyright (c) 2025, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "framework.test.h" +#include + +namespace lws { namespace db +{ + inline std::ostream& operator<<(std::ostream& out, const index_ranges& src) + { + using lest::to_string; + return out << to_string(src.get_container()); + } +}} // lws // db + diff --git a/tests/unit/db/storage.test.cpp b/tests/unit/db/storage.test.cpp index 28331aa..4c1dc9a 100644 --- a/tests/unit/db/storage.test.cpp +++ b/tests/unit/db/storage.test.cpp @@ -30,6 +30,9 @@ #include #include "common/util.h" // monero/src/ +#include "db/string.h" +#include "error.h" +#include "framework.test.h" namespace lws { namespace db { namespace test { @@ -68,3 +71,201 @@ namespace lws { namespace db { namespace test return lws::account{make_db_account(pubs, key), {}, {}}; } }}} // lws // db // test + +LWS_CASE("lws::db::storage") +{ + lws::db::account_address account_address{}; + crypto::secret_key view{}; + crypto::generate_keys(account_address.spend_public, view); + crypto::generate_keys(account_address.view_public, view); + const std::string address = lws::db::address_string(account_address); + const std::string viewkey = epee::to_hex::string(epee::as_byte_span(unwrap(unwrap(view)))); + + const lws::db::address_index lookahead{ + lws::db::major_index(2), lws::db::minor_index(2) + }; + + SETUP("Database with account") + { + lws::db::test::cleanup_db on_scope_exit{}; + lws::db::storage db = lws::db::test::get_fresh_db(); + EXPECT(db.add_account(account_address, view)); + + const lws::db::block_info last_block = + MONERO_UNWRAP(MONERO_UNWRAP(db.start_read()).get_last_block()); + + const auto get_account = [&db, &account_address] () -> lws::db::account + { + return MONERO_UNWRAP(MONERO_UNWRAP(db.start_read()).get_account(account_address)).second; + }; + + SECTION("rollback lookahead_fail via rescan") + { + EXPECT(db.import_request(account_address, last_block.id, lookahead)); + EXPECT(db.accept_requests(lws::db::request::import_scan, {std::addressof(account_address), 1}, 10)); + + const auto block_failed = lws::db::block_id(to_uint(last_block.id) + 1); + const auto update = + db.update_lookahead(account_address, block_failed, {lws::db::major_index(10), lws::db::minor_index(10)}, 10); + EXPECT(update == -1); + EXPECT(get_account().lookahead_fail == block_failed); + + EXPECT(db.rescan(last_block.id, {std::addressof(account_address), 1})); + EXPECT(get_account().lookahead_fail == lws::db::block_id(0)); + } + + SECTION("rollback lookahead_fail via import") + { + EXPECT(db.import_request(account_address, last_block.id, lookahead)); + EXPECT(db.accept_requests(lws::db::request::import_scan, {std::addressof(account_address), 1}, 10)); + + const auto block_failed = lws::db::block_id(to_uint(last_block.id) + 1); + const auto update = + db.update_lookahead(account_address, block_failed, {lws::db::major_index(10), lws::db::minor_index(10)}, 10); + EXPECT(update == -1); + EXPECT(get_account().lookahead_fail == block_failed); + + EXPECT(db.import_request(account_address, last_block.id, lookahead)); + EXPECT(db.accept_requests(lws::db::request::import_scan, {std::addressof(account_address), 1}, 10)); + EXPECT(get_account().lookahead_fail == lws::db::block_id(0)); + } + + const auto add_output = [&] () + { + auto account = get_account(); + const lws::db::transaction_link link{ + lws::db::block_id(to_uint(last_block.id) + 1), crypto::rand() + }; + const crypto::public_key tx_public = []() { + crypto::secret_key secret; + crypto::public_key out; + crypto::generate_keys(out, secret); + return out; + }(); + const crypto::hash tx_prefix = crypto::rand(); + const crypto::public_key pub = crypto::rand(); + const rct::key ringct = crypto::rand(); + const auto extra = + lws::db::extra(lws::db::extra::coinbase_output | lws::db::extra::ringct_output); + const auto payment_id_ = crypto::rand(); + const crypto::key_image image = crypto::rand(); + + lws::account real_account{account, {}, {}}; + real_account.add_out( + lws::db::output{ + link, + lws::db::output::spend_meta_{ + lws::db::output_id{500, 30}, + std::uint64_t(40000), + std::uint32_t(16), + std::uint32_t(2), + tx_public + }, + std::uint64_t(7000), + std::uint64_t(4670), + tx_prefix, + pub, + ringct, + {0, 0, 0, 0, 0, 0, 0}, + lws::db::pack(extra, sizeof(crypto::hash)), + payment_id_, + std::uint64_t(100), + lws::db::address_index{lws::db::major_index(2), lws::db::minor_index(10)} + } + ); + + { + std::vector hashes{ + last_block.hash, + crypto::rand(), + crypto::rand(), + crypto::rand(), + crypto::rand(), + crypto::rand() + }; + + const auto thing = db.update(last_block.id, epee::to_span(hashes), {std::addressof(real_account), 1}, {}); + if (!thing) + std::cout << thing.error().message() << std::endl; + } + }; + + + SECTION("Lookahead with outputs") + { + add_output(); + const auto scan_height = get_account().scan_height; + + EXPECT(MONERO_UNWRAP(MONERO_UNWRAP(db.start_read()).get_subaddresses(lws::db::account_id(1))).empty()); + EXPECT(db.import_request(account_address, scan_height, lookahead)); + EXPECT(db.accept_requests(lws::db::request::import_scan, {std::addressof(account_address), 1}, 18)); + + const std::vector expected_range{ + {lws::db::major_index(0), {{lws::db::index_range{lws::db::minor_index(0), lws::db::minor_index(1)}}}}, + {lws::db::major_index(1), {{lws::db::index_range{lws::db::minor_index(0), lws::db::minor_index(1)}}}}, + {lws::db::major_index(2), {{lws::db::index_range{lws::db::minor_index(0), lws::db::minor_index(11)}}}}, + {lws::db::major_index(3), {{lws::db::index_range{lws::db::minor_index(0), lws::db::minor_index(1)}}}}, + }; + EXPECT(MONERO_UNWRAP(MONERO_UNWRAP(db.start_read()).get_subaddresses(lws::db::account_id(1))) == expected_range); + + SECTION("shrink lookahead") + { + const lws::db::address_index test1{ + lws::db::major_index(3), lws::db::minor_index(2) + }; + const lws::db::address_index test2{ + lws::db::major_index(2), lws::db::minor_index(3) + }; + const lws::db::address_index shrink{ + lws::db::major_index(1), lws::db::minor_index(1) + }; + EXPECT(!db.shrink_lookahead(account_address, test1)); + EXPECT(!db.shrink_lookahead(account_address, test2)); + EXPECT(db.shrink_lookahead(account_address, lookahead)); + + EXPECT(MONERO_UNWRAP(MONERO_UNWRAP(db.start_read()).get_subaddresses(lws::db::account_id(1))) == expected_range); + EXPECT(get_account().lookahead == lookahead); + EXPECT(MONERO_UNWRAP(MONERO_UNWRAP(db.start_read()).get_subaddresses(lws::db::account_id(1))) == expected_range); + + EXPECT(db.shrink_lookahead(account_address, shrink)); + EXPECT(get_account().lookahead == shrink); + EXPECT(MONERO_UNWRAP(MONERO_UNWRAP(db.start_read()).get_subaddresses(lws::db::account_id(1))) == expected_range); + } + } + + SECTION("Lookahead failure with outputs") + { + add_output(); + const auto scan_height = get_account().scan_height; + + EXPECT(MONERO_UNWRAP(MONERO_UNWRAP(db.start_read()).get_subaddresses(lws::db::account_id(1))).empty()); + EXPECT(db.import_request(account_address, scan_height, lookahead)); + EXPECT(db.accept_requests(lws::db::request::import_scan, {std::addressof(account_address), 1}, 17)); + + const std::vector expected_range{ + {lws::db::major_index(0), {{lws::db::index_range{lws::db::minor_index(0), lws::db::minor_index(1)}}}}, + {lws::db::major_index(1), {{lws::db::index_range{lws::db::minor_index(0), lws::db::minor_index(1)}}}}, + {lws::db::major_index(2), {{lws::db::index_range{lws::db::minor_index(0), lws::db::minor_index(11)}}}} + }; + + EXPECT(MONERO_UNWRAP(MONERO_UNWRAP(db.start_read()).get_subaddresses(lws::db::account_id(1))) == expected_range); + + SECTION("shrink lookahead") + { + const lws::db::address_index test1{ + lws::db::major_index(3), lws::db::minor_index(2) + }; + const lws::db::address_index test2{ + lws::db::major_index(2), lws::db::minor_index(3) + }; + const lws::db::address_index shrink{ + lws::db::major_index(1), lws::db::minor_index(1) + }; + EXPECT(!db.shrink_lookahead(account_address, test1)); + EXPECT(!db.shrink_lookahead(account_address, test2)); + EXPECT(!db.shrink_lookahead(account_address, lookahead)); + EXPECT(!db.shrink_lookahead(account_address, shrink)); + } + } + } +} diff --git a/tests/unit/lest.hpp b/tests/unit/lest.hpp index f41942c..ad09806 100644 --- a/tests/unit/lest.hpp +++ b/tests/unit/lest.hpp @@ -655,6 +655,9 @@ using ForContainer = typename std::enable_if< is_container::value, R>::type; template< typename T, typename R > using ForNonContainerNonPointer = typename std::enable_if< ! (is_container::value || std::is_pointer::value), R>::type; +template< typename T > +auto to_string( T const & item ) -> ForNonContainerNonPointer; + template< typename T > auto make_enum_string( T const & item ) -> ForNonEnum { diff --git a/tests/unit/rest.test.cpp b/tests/unit/rest.test.cpp index 281c418..2243c24 100644 --- a/tests/unit/rest.test.cpp +++ b/tests/unit/rest.test.cpp @@ -29,6 +29,7 @@ #include #include "db/data.h" +#include "db/print.test.h" #include "db/storage.test.h" #include "db/string.h" #include "error.h" @@ -151,7 +152,7 @@ LWS_CASE("rest_server") std::string message = "{\"address\":\"" + address + "\",\"view_key\":\"" + viewkey + "\",\"create_account\":true,\"generated_locally\":true}"; std::string response = invoke(client, "/login", message); - EXPECT(response == "{\"new_address\":true,\"generated_locally\":true}"); + EXPECT(response == "{\"new_address\":true,\"generated_locally\":true,\"lookahead\":{\"maj_i\":0,\"min_i\":0}}"); auto account = get_account(); EXPECT(account.id == lws::db::account_id(1)); @@ -219,10 +220,20 @@ LWS_CASE("rest_server") "{\"import_fee\":\"0\"," "\"status\":\"Accepted, waiting for approval\"," "\"new_request\":true," - "\"request_fulfilled\":false}" + "\"request_fulfilled\":false," + "\"lookahead\":{\"maj_i\":0,\"min_i\":0}}" + ); + + EXPECT(db.accept_requests(lws::db::request::import_scan, {std::addressof(account_address), 1}, 0)); + response = invoke(client, "/import_wallet_request", message); + EXPECT(response == + "{\"import_fee\":\"0\"," + "\"status\":\"Approved\"," + "\"new_request\":false," + "\"request_fulfilled\":true," + "\"lookahead\":{\"maj_i\":0,\"min_i\":0}}" ); - EXPECT(db.accept_requests(lws::db::request::import_scan, {std::addressof(account_address), 1})); response = invoke(client, "/get_address_info", message); EXPECT(response == "{\"locked_funds\":\"0\"," @@ -236,6 +247,167 @@ LWS_CASE("rest_server") ); } + SECTION("Import with lookahead") + { + EXPECT(account.start_height != lws::db::block_id(0)); + + const std::string scan_height = std::to_string(std::uint64_t(account.scan_height)); + const std::string start_height = std::to_string(std::uint64_t(account.start_height)); + message = "{\"address\":\"" + address + "\",\"view_key\":\"" + viewkey + "\"}"; + response = invoke(client, "/get_address_info", message); + EXPECT(response == + "{\"locked_funds\":\"0\"," + "\"total_received\":\"0\"," + "\"total_sent\":\"0\"," + "\"scanned_height\":" + scan_height + "," + + "\"scanned_block_height\":" + scan_height + "," + "\"start_height\":" + start_height + "," + "\"transaction_height\":" + scan_height + "," + "\"blockchain_height\":" + scan_height + "}" + ); + + message = "{\"address\":\"" + address + "\",\"view_key\":\"" + viewkey + "\", \"lookahead\":{\"maj_i\":2,\"min_i\":3}}"; + response = invoke(client, "/import_wallet_request", message); + EXPECT(response == + "{\"import_fee\":\"0\"," + "\"status\":\"Accepted, waiting for approval\"," + "\"new_request\":true," + "\"request_fulfilled\":false," + "\"lookahead\":{\"maj_i\":0,\"min_i\":0}}" + ); + + response = invoke(client, "/import_wallet_request", message); + EXPECT(response == + "{\"import_fee\":\"0\"," + "\"status\":\"Waiting for Approval\"," + "\"new_request\":false," + "\"request_fulfilled\":false," + "\"lookahead\":{\"maj_i\":0,\"min_i\":0}}" + ); + + { + auto reader = MONERO_UNWRAP(db.start_read()); + const std::vector expected_range{}; + EXPECT(MONERO_UNWRAP(reader.get_subaddresses(lws::db::account_id(1))) == expected_range); + } + + EXPECT(db.accept_requests(lws::db::request::import_scan, {std::addressof(account_address), 1}, 6)); + response = invoke(client, "/import_wallet_request", message); + EXPECT(response == + "{\"import_fee\":\"0\"," + "\"status\":\"Approved\"," + "\"new_request\":false," + "\"request_fulfilled\":true," + "\"lookahead\":{\"maj_i\":2,\"min_i\":3}}" + ); + + response = invoke(client, "/get_address_info", message); + EXPECT(response == + "{\"locked_funds\":\"0\"," + "\"total_received\":\"0\"," + "\"total_sent\":\"0\"," + "\"scanned_height\":0," + "\"scanned_block_height\":0," + "\"start_height\":0," + "\"transaction_height\":" + scan_height + "," + "\"blockchain_height\":" + scan_height + "," + "\"lookahead\":{\"maj_i\":2,\"min_i\":3}}" + ); + + { + auto reader = MONERO_UNWRAP(db.start_read()); + const auto account = MONERO_UNWRAP(reader.get_account(lws::db::account_status::active, lws::db::account_id(1))); + const lws::db::address_index lookahead{lws::db::major_index(2), lws::db::minor_index(3)}; + EXPECT(account.lookahead == lookahead); + EXPECT(account.lookahead_fail == lws::db::block_id(0)); + + const std::vector expected_range{ + {lws::db::major_index(0), {{lws::db::index_range{lws::db::minor_index(0), lws::db::minor_index(2)}}}}, + {lws::db::major_index(1), {{lws::db::index_range{lws::db::minor_index(0), lws::db::minor_index(2)}}}}, + }; + EXPECT(MONERO_UNWRAP(reader.get_subaddresses(lws::db::account_id(1))) == expected_range); + } + } + + SECTION("Import with lookahead failure") + { + EXPECT(account.start_height != lws::db::block_id(0)); + + const std::string scan_height = std::to_string(std::uint64_t(account.scan_height)); + const std::string start_height = std::to_string(std::uint64_t(account.start_height)); + message = "{\"address\":\"" + address + "\",\"view_key\":\"" + viewkey + "\"}"; + response = invoke(client, "/get_address_info", message); + EXPECT(response == + "{\"locked_funds\":\"0\"," + "\"total_received\":\"0\"," + "\"total_sent\":\"0\"," + "\"scanned_height\":" + scan_height + "," + + "\"scanned_block_height\":" + scan_height + "," + "\"start_height\":" + start_height + "," + "\"transaction_height\":" + scan_height + "," + "\"blockchain_height\":" + scan_height + "}" + ); + + message = "{\"address\":\"" + address + "\",\"view_key\":\"" + viewkey + "\", \"lookahead\":{\"maj_i\":2,\"min_i\":3}}"; + response = invoke(client, "/import_wallet_request", message); + EXPECT(response == + "{\"import_fee\":\"0\"," + "\"status\":\"Accepted, waiting for approval\"," + "\"new_request\":true," + "\"request_fulfilled\":false," + "\"lookahead\":{\"maj_i\":0,\"min_i\":0}}" + ); + + response = invoke(client, "/import_wallet_request", message); + EXPECT(response == + "{\"import_fee\":\"0\"," + "\"status\":\"Waiting for Approval\"," + "\"new_request\":false," + "\"request_fulfilled\":false," + "\"lookahead\":{\"maj_i\":0,\"min_i\":0}}" + ); + + { + auto reader = MONERO_UNWRAP(db.start_read()); + const std::vector expected_range{}; + EXPECT(MONERO_UNWRAP(reader.get_subaddresses(lws::db::account_id(1))) == expected_range); + } + + EXPECT(db.accept_requests(lws::db::request::import_scan, {std::addressof(account_address), 1}, 5)); + response = invoke(client, "/import_wallet_request", message); + EXPECT(response == + "{\"import_fee\":\"0\"," + "\"status\":\"Accepted, waiting for approval\"," + "\"new_request\":true," + "\"request_fulfilled\":false," + "\"lookahead\":{\"maj_i\":2,\"min_i\":3}}" + ); + + response = invoke(client, "/get_address_info", message); + EXPECT(response == + "{\"locked_funds\":\"0\"," + "\"total_received\":\"0\"," + "\"total_sent\":\"0\"," + "\"scanned_height\":0," + "\"scanned_block_height\":0," + "\"start_height\":0," + "\"transaction_height\":" + scan_height + "," + "\"blockchain_height\":" + scan_height + "," + "\"lookahead_fail\":1," + "\"lookahead\":{\"maj_i\":2,\"min_i\":3}}" + ); + + { + auto reader = MONERO_UNWRAP(db.start_read()); + const auto account = MONERO_UNWRAP(reader.get_account(lws::db::account_status::active, lws::db::account_id(1))); + const lws::db::address_index lookahead{lws::db::major_index(2), lws::db::minor_index(3)}; + EXPECT(account.lookahead == lookahead); + EXPECT(account.lookahead_fail == lws::db::block_id(1)); + EXPECT(MONERO_UNWRAP(reader.get_subaddresses(lws::db::account_id(1))).empty()); + } + } + + SECTION("One Receive, Zero Spends") { const std::string scan_height = std::to_string(std::uint64_t(account.scan_height) + 5); diff --git a/tests/unit/scanner.test.cpp b/tests/unit/scanner.test.cpp index df5f100..235c071 100644 --- a/tests/unit/scanner.test.cpp +++ b/tests/unit/scanner.test.cpp @@ -35,6 +35,7 @@ #include "cryptonote_config.h" // monero/src #include "cryptonote_core/cryptonote_tx_utils.h" // monero/src #include "db/chain.test.h" +#include "db/print.test.h" #include "db/storage.test.h" #include "device/device_default.hpp" // monero/src #include "hardforks/hardforks.h" // monero/src @@ -149,7 +150,7 @@ namespace { pub_keys.push_back(val.key); } }; - transaction make_tx(lest::env& lest_env, const cryptonote::account_keys& keys, std::vector& destinations, const std::uint32_t ring_base, const bool use_view_tag) + transaction make_tx(lest::env& lest_env, const cryptonote::account_keys& keys, std::vector destinations, const std::uint32_t ring_base, const bool use_view_tag) { static constexpr std::uint64_t input_amount = 20000; static constexpr std::uint64_t output_amount = 8000; @@ -174,6 +175,9 @@ namespace subaddresses[destination.addr.m_spend_public_key] = {0, index}; } + if (2 < destinations.size()) + destinations.erase(destinations.begin() + 1, destinations.end() - 1); + std::vector sources; sources.emplace_back(); sources.back().amount = input_amount; @@ -307,7 +311,7 @@ LWS_CASE("lws::scanner::sync and lws::scanner::run") keys_subaddr2.m_account_address = hw.get_subaddress(keys, cryptonote::subaddress_index{0, 2}); const auto sub1_secret = hw.get_subaddress_secret_key(keys.m_view_secret_key, cryptonote::subaddress_index{0, 1}); - const auto sub2_secret = hw.get_subaddress_secret_key(keys.m_view_secret_key, cryptonote::subaddress_index{0, 1}); + const auto sub2_secret = hw.get_subaddress_secret_key(keys.m_view_secret_key, cryptonote::subaddress_index{0, 2}); sc_add(to_bytes(keys_subaddr1.m_spend_secret_key), to_bytes(sub1_secret), to_bytes(keys.m_spend_secret_key)); sc_add(to_bytes(keys_subaddr1.m_view_secret_key), to_bytes(keys_subaddr1.m_spend_secret_key), to_bytes(keys.m_view_secret_key)); @@ -416,7 +420,7 @@ LWS_CASE("lws::scanner::sync and lws::scanner::run") } } - SECTION("lws::scanner::run") + SECTION("lws::scanner::run (with upsert)") { { const std::vector indexes{ @@ -436,7 +440,7 @@ LWS_CASE("lws::scanner::sync and lws::scanner::run") EXPECT(result->at(0).second.get_container().at(0).size() == 2); EXPECT(result->at(0).second.get_container().at(0).at(0) == lws::db::minor_index(1)); EXPECT(result->at(0).second.get_container().at(0).at(1) == lws::db::minor_index(2)); - } + } std::vector destinations; destinations.emplace_back(); @@ -534,7 +538,7 @@ LWS_CASE("lws::scanner::sync and lws::scanner::run") bmessage.output_indices.resize(1); messages.push_back(daemon_response(bmessage)); { - static constexpr const lws::scanner_options opts{true, false}; + static constexpr const lws::scanner_options opts{1, false}; lws::scanner scanner{db.clone(), epee::net_utils::ssl_verification_t::none}; boost::thread server_thread(&scanner_thread, std::ref(scanner), rpc.zmq_context(), std::cref(messages)); const join on_scope_exit{server_thread}; @@ -735,7 +739,412 @@ LWS_CASE("lws::scanner::sync and lws::scanner::run") EXPECT(MONERO_UNWRAP(reader.get_outputs(lws::db::account_id(2))).count() == 0); EXPECT(MONERO_UNWRAP(reader.get_spends(lws::db::account_id(2))).count() == 0); } - } //SECTION (lws::scanner::run) + } //SECTION (lws::scanner::run (with upsert)) + + SECTION("lws::scanner::run (with lookahead)") + { + std::vector destinations; + destinations.emplace_back(); + destinations.back().amount = 8000; + destinations.back().addr = keys.m_account_address; + + std::vector messages{}; + transaction tx = make_miner_tx(lest_env, last_block.id, account, false); + EXPECT(tx.pub_keys.size() == 1); + EXPECT(tx.spend_publics.size() == 1); + + transaction tx2 = make_tx(lest_env, keys, destinations, 20, true); + EXPECT(tx2.pub_keys.size() == 1); + EXPECT(tx2.spend_publics.size() == 1); + + transaction tx3 = make_tx(lest_env, keys, destinations, 86, false); + EXPECT(tx3.pub_keys.size() == 1); + EXPECT(tx3.spend_publics.size() == 1); + + destinations.emplace_back(); + destinations.back().amount = 2000; + destinations.back().addr = keys_subaddr1.m_account_address; + destinations.back().is_subaddress = true; + + transaction tx4 = make_tx(lest_env, keys, destinations, 50, false); + EXPECT(tx4.pub_keys.size() == 1); + EXPECT(tx4.spend_publics.size() == 2); + + destinations.emplace_back(); + destinations.back().amount = 1000; + destinations.back().addr = keys_subaddr2.m_account_address; + destinations.back().is_subaddress = true; + + transaction tx5 = make_tx(lest_env, keys, destinations, 146, true); + EXPECT(tx5.pub_keys.size() == 1); + EXPECT(tx5.spend_publics.size() == 2); + + cryptonote::rpc::GetBlocksFast::Response bmessage{}; + bmessage.start_height = std::uint64_t(last_block.id) + 1; + bmessage.current_height = bmessage.start_height + 1; + bmessage.blocks.emplace_back(); + bmessage.blocks.back().block.miner_tx = tx.tx; + bmessage.blocks.back().block.tx_hashes.push_back(cryptonote::get_transaction_hash(tx2.tx)); + bmessage.blocks.back().block.tx_hashes.push_back(cryptonote::get_transaction_hash(tx3.tx)); + bmessage.blocks.back().block.tx_hashes.push_back(cryptonote::get_transaction_hash(tx4.tx)); + bmessage.blocks.back().block.tx_hashes.push_back(cryptonote::get_transaction_hash(tx5.tx)); + bmessage.blocks.back().transactions.push_back(tx2.tx); + bmessage.blocks.back().transactions.push_back(tx3.tx); + bmessage.blocks.back().transactions.push_back(tx4.tx); + bmessage.blocks.back().transactions.push_back(tx5.tx); + bmessage.output_indices.emplace_back(); + bmessage.output_indices.back().emplace_back(); + bmessage.output_indices.back().back().push_back(100); + bmessage.output_indices.back().emplace_back(); + bmessage.output_indices.back().back().push_back(101); + bmessage.output_indices.back().emplace_back(); + bmessage.output_indices.back().back().push_back(102); + bmessage.output_indices.back().emplace_back(); + bmessage.output_indices.back().back().push_back(200); + bmessage.output_indices.back().back().push_back(201); + bmessage.output_indices.back().emplace_back(); + bmessage.output_indices.back().back().push_back(300); + bmessage.output_indices.back().back().push_back(301); + bmessage.blocks.push_back(bmessage.blocks.back()); + bmessage.output_indices.push_back(bmessage.output_indices.back()); + + std::vector hashes{ + last_block.hash, + cryptonote::get_block_hash(bmessage.blocks.back().block), + }; + { + cryptonote::rpc::GetHashesFast::Response hmessage{}; + + hmessage.start_height = std::uint64_t(last_block.id); + hmessage.hashes = hashes; + hmessage.current_height = hmessage.start_height + hashes.size() - 1; + messages.push_back(daemon_response(hmessage)); + + hmessage.start_height = hmessage.current_height; + hmessage.hashes.front() = hmessage.hashes.back(); + hmessage.hashes.resize(1); + messages.push_back(daemon_response(hmessage)); + + { + lws::scanner scanner{db.clone(), epee::net_utils::ssl_verification_t::none}; + boost::thread server_thread(&scanner_thread, std::ref(scanner), rpc.zmq_context(), std::cref(messages)); + const join on_scope_exit{server_thread}; + EXPECT(scanner.sync(MONERO_UNWRAP(rpc.connect()))); + lws_test::test_chain(lest_env, MONERO_UNWRAP(db.start_read()), last_block.id, epee::to_span(hashes)); + } + } + + EXPECT(db.add_account(account, keys.m_view_secret_key)); + EXPECT(db.add_account(account2, keys2.m_view_secret_key)); + + { + auto reader = MONERO_UNWRAP(db.start_read()); + const std::vector expected_range{}; + EXPECT(MONERO_UNWRAP(reader.get_subaddresses(lws::db::account_id(1))) == expected_range); + } + + const lws::db::block_id user_height = + MONERO_UNWRAP(MONERO_UNWRAP(db.start_read()).get_account(lws::db::account_status::active, lws::db::account_id(1))).scan_height; + + EXPECT(db.import_request(account, user_height, {lws::db::major_index(1), lws::db::minor_index(2)})); + EXPECT(db.accept_requests(lws::db::request::import_scan, {std::addressof(account), 1}, 2)); + + { + auto reader = MONERO_UNWRAP(db.start_read()); + const std::vector expected_range{ + {lws::db::major_index(0), {{lws::db::index_range{lws::db::minor_index(0), lws::db::minor_index(1)}}}} + }; + EXPECT(MONERO_UNWRAP(reader.get_subaddresses(lws::db::account_id(1))) == expected_range); + } + + messages.clear(); + messages.push_back(daemon_response(bmessage)); + bmessage.start_height = bmessage.current_height; + bmessage.blocks.resize(1); + bmessage.output_indices.resize(1); + messages.push_back(daemon_response(bmessage)); + { + static constexpr const lws::scanner_options opts{10, false}; + lws::scanner scanner{db.clone(), epee::net_utils::ssl_verification_t::none}; + boost::thread server_thread(&scanner_thread, std::ref(scanner), rpc.zmq_context(), std::cref(messages)); + const join on_scope_exit{server_thread}; + scanner.run(std::move(rpc), 1, {}, {}, opts); + } + + hashes.push_back(cryptonote::get_block_hash(bmessage.blocks.back().block)); + lws_test::test_chain(lest_env, MONERO_UNWRAP(db.start_read()), last_block.id, epee::to_span(hashes)); + + const lws::db::block_id new_last_block_id = lws::db::block_id(std::uint64_t(last_block.id) + 2); + EXPECT(get_account().scan_height == new_last_block_id); + { + const std::map, lws::db::output> expected{ + { + {lws::db::output_id{0, 100}, 35184372088830}, lws::db::output{ + lws::db::transaction_link{new_last_block_id, cryptonote::get_transaction_hash(tx.tx)}, + lws::db::output::spend_meta_{ + lws::db::output_id{0, 100}, 35184372088830, 0, 0, tx.pub_keys.at(0) + }, + 0, + 0, + cryptonote::get_transaction_prefix_hash(tx.tx), + tx.spend_publics.at(0), + rct::commit(35184372088830, rct::identity()), + {}, + lws::db::pack(lws::db::extra(lws::db::extra::coinbase_output | lws::db::extra::ringct_output), 0), + {}, + 0, // fee + lws::db::address_index{} + }, + }, + { + {lws::db::output_id{0, 101}, 8000}, lws::db::output{ + lws::db::transaction_link{new_last_block_id, cryptonote::get_transaction_hash(tx2.tx)}, + lws::db::output::spend_meta_{ + lws::db::output_id{0, 101}, 8000, 15, 0, tx2.pub_keys.at(0) + }, + 0, + 0, + cryptonote::get_transaction_prefix_hash(tx2.tx), + tx2.spend_publics.at(0), + tx2.tx.rct_signatures.outPk.at(0).mask, + {}, + lws::db::pack(lws::db::extra::ringct_output, 8), + {}, + 12000, // fee + lws::db::address_index{} + }, + }, + { + {lws::db::output_id{0, 102}, 8000}, lws::db::output{ + lws::db::transaction_link{new_last_block_id, cryptonote::get_transaction_hash(tx3.tx)}, + lws::db::output::spend_meta_{ + lws::db::output_id{0, 102}, 8000, 15, 0, tx3.pub_keys.at(0) + }, + 0, + 0, + cryptonote::get_transaction_prefix_hash(tx3.tx), + tx3.spend_publics.at(0), + tx3.tx.rct_signatures.outPk.at(0).mask, + {}, + lws::db::pack(lws::db::extra::ringct_output, 8), + {}, + 12000, // fee + lws::db::address_index{} + }, + }, + { + {lws::db::output_id{0, 200}, 8000}, lws::db::output{ + lws::db::transaction_link{new_last_block_id, cryptonote::get_transaction_hash(tx4.tx)}, + lws::db::output::spend_meta_{ + lws::db::output_id{0, 200}, 8000, 15, 0, tx4.pub_keys.at(0) + }, + 0, + 0, + cryptonote::get_transaction_prefix_hash(tx4.tx), + tx4.spend_publics.at(0), + tx4.tx.rct_signatures.outPk.at(0).mask, + {}, + lws::db::pack(lws::db::extra::ringct_output, 8), + {}, + 10000, // fee + lws::db::address_index{} + } + }, + { + {lws::db::output_id{0, 201}, 8000}, lws::db::output{ + lws::db::transaction_link{new_last_block_id, cryptonote::get_transaction_hash(tx4.tx)}, + lws::db::output::spend_meta_{ + lws::db::output_id{0, 201}, 8000, 15, 1, tx4.pub_keys.at(0) + }, + 0, + 0, + cryptonote::get_transaction_prefix_hash(tx4.tx), + tx4.spend_publics.at(1), + tx4.tx.rct_signatures.outPk.at(1).mask, + {}, + lws::db::pack(lws::db::extra::ringct_output, 8), + {}, + 10000, // fee + lws::db::address_index{} + } + }, + { + {lws::db::output_id{0, 200}, 2000}, lws::db::output{ + lws::db::transaction_link{new_last_block_id, cryptonote::get_transaction_hash(tx4.tx)}, + lws::db::output::spend_meta_{ + lws::db::output_id{0, 200}, 2000, 15, 0, tx4.pub_keys.at(0) + }, + 0, + 0, + cryptonote::get_transaction_prefix_hash(tx4.tx), + tx4.spend_publics.at(0), + tx4.tx.rct_signatures.outPk.at(0).mask, + {}, + lws::db::pack(lws::db::extra::ringct_output, 8), + {}, + 10000, // fee + lws::db::address_index{lws::db::major_index::primary, lws::db::minor_index(1)} + } + }, + { + {lws::db::output_id{0, 201}, 2000}, lws::db::output{ + lws::db::transaction_link{new_last_block_id, cryptonote::get_transaction_hash(tx4.tx)}, + lws::db::output::spend_meta_{ + lws::db::output_id{0, 201}, 2000, 15, 1, tx4.pub_keys.at(0) + }, + 0, + 0, + cryptonote::get_transaction_prefix_hash(tx4.tx), + tx4.spend_publics.at(1), + tx4.tx.rct_signatures.outPk.at(1).mask, + {}, + lws::db::pack(lws::db::extra::ringct_output, 8), + {}, + 10000, // fee + lws::db::address_index{lws::db::major_index::primary, lws::db::minor_index(1)} + } + }, + { + {lws::db::output_id{0, 300}, 8000}, lws::db::output{ + lws::db::transaction_link{new_last_block_id, cryptonote::get_transaction_hash(tx5.tx)}, + lws::db::output::spend_meta_{ + lws::db::output_id{0, 300}, 8000, 15, 0, tx5.pub_keys.at(0) + }, + 0, + 0, + cryptonote::get_transaction_prefix_hash(tx5.tx), + tx5.spend_publics.at(0), + tx5.tx.rct_signatures.outPk.at(0).mask, + {}, + lws::db::pack(lws::db::extra::ringct_output, 8), + {}, + 11000, // fee + lws::db::address_index{} + } + }, + { + {lws::db::output_id{0, 301}, 8000}, lws::db::output{ + lws::db::transaction_link{new_last_block_id, cryptonote::get_transaction_hash(tx5.tx)}, + lws::db::output::spend_meta_{ + lws::db::output_id{0, 301}, 8000, 15, 1, tx5.pub_keys.at(0) + }, + 0, + 0, + cryptonote::get_transaction_prefix_hash(tx5.tx), + tx5.spend_publics.at(1), + tx5.tx.rct_signatures.outPk.at(1).mask, + {}, + lws::db::pack(lws::db::extra::ringct_output, 8), + {}, + 11000, // fee + lws::db::address_index{} + } + }, + { + {lws::db::output_id{0, 300}, 1000}, lws::db::output{ + lws::db::transaction_link{new_last_block_id, cryptonote::get_transaction_hash(tx5.tx)}, + lws::db::output::spend_meta_{ + lws::db::output_id{0, 300}, 1000, 15, 0, tx5.pub_keys.at(0) + }, + 0, + 0, + cryptonote::get_transaction_prefix_hash(tx5.tx), + tx5.spend_publics.at(0), + tx5.tx.rct_signatures.outPk.at(0).mask, + {}, + lws::db::pack(lws::db::extra::ringct_output, 8), + {}, + 11000, // fee + lws::db::address_index{lws::db::major_index::primary, lws::db::minor_index(2)} + } + }, + { + {lws::db::output_id{0, 301}, 1000}, lws::db::output{ + lws::db::transaction_link{new_last_block_id, cryptonote::get_transaction_hash(tx5.tx)}, + lws::db::output::spend_meta_{ + lws::db::output_id{0, 301}, 1000, 15, 1, tx5.pub_keys.at(0) + }, + 0, + 0, + cryptonote::get_transaction_prefix_hash(tx5.tx), + tx5.spend_publics.at(1), + tx5.tx.rct_signatures.outPk.at(1).mask, + {}, + lws::db::pack(lws::db::extra::ringct_output, 8), + {}, + 11000, // fee + lws::db::address_index{lws::db::major_index::primary, lws::db::minor_index(2)} + } + } + }; + + auto reader = MONERO_UNWRAP(db.start_read()); + auto outputs = MONERO_UNWRAP(reader.get_outputs(lws::db::account_id(1))); + EXPECT(outputs.count() == 6); + auto output_it = outputs.make_iterator(); + for (auto output_it = outputs.make_iterator(); !output_it.is_end(); ++output_it) + { + auto real_output = *output_it; + const auto expected_output = + expected.find(std::make_pair(real_output.spend_meta.id, real_output.spend_meta.amount)); + EXPECT(expected_output != expected.end()); + + EXPECT(real_output.link.height == expected_output->second.link.height); + EXPECT(real_output.link.tx_hash == expected_output->second.link.tx_hash); + EXPECT(real_output.spend_meta.id == expected_output->second.spend_meta.id); + EXPECT(real_output.spend_meta.amount == expected_output->second.spend_meta.amount); + EXPECT(real_output.spend_meta.mixin_count == expected_output->second.spend_meta.mixin_count); + EXPECT(real_output.spend_meta.index == expected_output->second.spend_meta.index); + EXPECT(real_output.tx_prefix_hash == expected_output->second.tx_prefix_hash); + EXPECT(real_output.spend_meta.tx_public == expected_output->second.spend_meta.tx_public); + EXPECT(real_output.pub == expected_output->second.pub); + EXPECT(rct::commit(real_output.spend_meta.amount, real_output.ringct_mask) == expected_output->second.ringct_mask); + EXPECT(real_output.extra == expected_output->second.extra); + if (unpack(expected_output->second.extra).second == 8) + EXPECT(real_output.payment_id.short_ == expected_output->second.payment_id.short_); + EXPECT(real_output.fee == expected_output->second.fee); + EXPECT(real_output.recipient == expected_output->second.recipient); + } + + auto spends = MONERO_UNWRAP(reader.get_spends(lws::db::account_id(1))); + EXPECT(spends.count() == 2); + auto spend_it = spends.make_iterator(); + EXPECT(!spend_it.is_end()); + + auto real_spend = *spend_it; + EXPECT(real_spend.link.height == new_last_block_id); + EXPECT(real_spend.link.tx_hash == cryptonote::get_transaction_hash(tx3.tx)); + lws::db::output_id expected_out{0, 100}; + EXPECT(real_spend.source == expected_out); + EXPECT(real_spend.mixin_count == 15); + EXPECT(real_spend.length == 0); + EXPECT(real_spend.payment_id == crypto::hash{}); + EXPECT(real_spend.sender == lws::db::address_index{}); + + ++spend_it; + EXPECT(!spend_it.is_end()); + + real_spend = *spend_it; + EXPECT(real_spend.link.height == new_last_block_id); + EXPECT(real_spend.link.tx_hash == cryptonote::get_transaction_hash(tx3.tx)); + expected_out = lws::db::output_id{0, 101}; + EXPECT(real_spend.source == expected_out); + EXPECT(real_spend.mixin_count == 15); + EXPECT(real_spend.length == 0); + EXPECT(real_spend.payment_id == crypto::hash{}); + EXPECT(real_spend.sender == lws::db::address_index{}); + + EXPECT(MONERO_UNWRAP(reader.get_outputs(lws::db::account_id(2))).count() == 0); + EXPECT(MONERO_UNWRAP(reader.get_spends(lws::db::account_id(2))).count() == 0); + + { + const std::vector expected_range{ + {lws::db::major_index(0), {{lws::db::index_range{lws::db::minor_index(0), lws::db::minor_index(2)}}}} + }; + EXPECT(MONERO_UNWRAP(reader.get_subaddresses(lws::db::account_id(1))) == expected_range); + } + } + } //SECTION (lws::scanner::run (lookahead)) } // SETUP } // LWS_CASE