such-gitea/wownero-lws
1
0
Fork 1
mirror of https://codeberg.org/wownero/wownero-lws synced 2026-01-09 15:15:15 -08:00
Code Issues Packages Projects Releases Wiki Activity

Add support for subaddress lookahead (#195)

Browse Source
This commit is contained in:
Lee *!* Clagett
2025-12-04 14:29:41 -05:00
committed by Lee *!* Clagett
parent e8b889e95f
commit 16111cae2c
20 changed files with 1588 additions and 210 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
src/admin_main.cpp
View File

@@ -96,12 +96,14 @@ namespace
struct options : lws::options
{
const command_line::arg_descriptor<bool> show_sensitive;
const command_line::arg_descriptor<std::uint32_t> max_subaddresses;
const command_line::arg_descriptor<std::string> command;
const command_line::arg_descriptor<std::vector<std::string>> arguments;
options()
: lws::options()
, show_sensitive{"show-sensitive", "Show view keys", false}
, max_subaddresses{"max-subaddresses", "Max subaddresses allowed on import/create", 0}
, command{"command", "Admin command to execute", ""}
, arguments{"arguments", "Arguments to command"}
{}
@@ -110,6 +112,7 @@ namespace
{
lws::options::prepare(description);
command_line::add_arg(description, show_sensitive);
command_line::add_arg(description, max_subaddresses);
command_line::add_arg(description, command);
command_line::add_arg(description, arguments);
}
@@ -119,6 +122,7 @@ namespace
{
lws::db::storage disk;
std::vector<std::string> arguments;
std::uint32_t max_subaddresses;
bool show_sensitive;
};
@@ -154,6 +158,7 @@ namespace
lws::rpc::address_requests req{
get_addresses(epee::to_span(prog.arguments)),
prog.max_subaddresses,
MONERO_UNWRAP(lws::db::request_from_string(prog.arguments[0]))
};
run_command(lws::rpc::accept_requests, out, std::move(prog.disk), std::move(req));
@@ -250,6 +255,7 @@ namespace
lws::rpc::address_requests req{
get_addresses(epee::to_span(prog.arguments)),
prog.max_subaddresses,
lws::db::request_from_string(prog.arguments[0]).value()
};
run_command(lws::rpc::reject_requests, out, std::move(prog.disk), std::move(req));
@@ -381,6 +387,7 @@ namespace
lws::db::storage::open(command_line::get_arg(args, opts.db_path).c_str(), 0)
};
prog.max_subaddresses = command_line::get_arg(args, opts.max_subaddresses);
prog.show_sensitive = command_line::get_arg(args, opts.show_sensitive);
auto cmd = args[opts.command.name];
if (cmd.empty())

7
src/db/data.cpp
View File

@@ -189,7 +189,9 @@ namespace db
WIRE_FIELD(start_height),
wire::field("creation_time", self.creation),
wire::field("admin", admin),
wire::field("generated_locally", generated_locally)
wire::field("generated_locally", generated_locally),
WIRE_FIELD(lookahead),
WIRE_FIELD(lookahead_fail)
);
}
@@ -403,7 +405,8 @@ namespace db
WIRE_FIELD(address),
wire::optional_field("view_key", key),
WIRE_FIELD(start_height),
wire::field("generated_locally", generated)
wire::field("generated_locally", generated),
WIRE_FIELD(lookahead)
);
}

17
src/db/data.h
View File

@@ -134,10 +134,16 @@ namespace db
enum class major_index : std::uint32_t { primary = 0 };
WIRE_AS_INTEGER(major_index);
inline constexpr std::uint32_t to_uint(const major_index src) noexcept
{ return std::uint32_t(src); }
//! Minor index of a subaddress
enum class minor_index : std::uint32_t { primary = 0 };
WIRE_AS_INTEGER(minor_index);
inline constexpr std::uint32_t to_uint(const minor_index src) noexcept
{ return std::uint32_t(src); }
//! Range within a major index
using index_range = std::array<minor_index, 2>;
@@ -185,8 +191,10 @@ namespace db
account_time creation; //!< Time account first appeared in database.
account_flags flags; //!< Additional account info bitmask.
char reserved[3];
address_index lookahead;
block_id lookahead_fail;
};
static_assert(sizeof(account) == (4 * 2) + 64 + 32 + (8 * 2) + (4 * 2), "padding in account");
static_assert(sizeof(account) == (4 * 2) + 64 + 32 + (8 * 2) + (4 * 2) + (4 * 2) + 8, "padding in account");
void write_bytes(wire::writer&, const account&, bool show_key = false);
//! Used with quick and full sync mode
@@ -331,8 +339,9 @@ namespace db
account_time creation; //!< Time the request was created.
account_flags creation_flags; //!< Generated locally?
char reserved[3];
address_index lookahead; //!< Desired subaddress lookahead
};
static_assert(sizeof(request_info) == 64 + 32 + 8 + (4 * 2), "padding in request_info");
static_assert(sizeof(request_info) == 64 + 32 + 8 + (4 * 2) + (4*2), "padding in request_info");
void write_bytes(wire::writer& dest, const request_info& self, bool show_key = false);
enum class webhook_type : std::uint8_t
@@ -428,6 +437,10 @@ namespace db
{
return left.maj_i == right.maj_i && left.min_i == right.min_i;
}
inline constexpr bool operator!=(address_index const& left, address_index const& right) noexcept
{
return left.maj_i != right.maj_i || left.min_i != right.min_i;
}
inline constexpr bool operator<(address_index const& left, address_index const& right) noexcept
{

704
src/db/storage.cpp
View File

@@ -80,7 +80,22 @@ namespace lws
namespace db
{
namespace v0
{
{
//! Original DB value, no lookahead
struct account
{
account_id id; //!< Must be first for LMDB optimizations
account_time access; //!< Last time `get_address_info` was called.
account_address address;
view_key key; //!< Doubles as authorization handle for REST API.
block_id scan_height; //!< Last block scanned; check-ins are always by block
block_id start_height; //!< Account started scanning at this block height
account_time creation; //!< Time account first appeared in database.
account_flags flags; //!< Additional account info bitmask.
char reserved[3];
};
static_assert(sizeof(account) == (4 * 2) + 64 + 32 + (8 * 2) + (4 * 2), "padding in account");
//! Orignal DB value, with no txn fee
struct output
{
@@ -130,6 +145,17 @@ namespace db
crypto::hash payment_id; //!< Unencrypted only, can't decrypt spend
};
static_assert(sizeof(spend) == 8 + 32 * 2 + 8 * 4 + 4 + 3 + 1 + 32, "padding in spend");
struct request_info
{
account_address address;//!< Must be first for LMDB optimizations
view_key key;
block_id start_height;
account_time creation; //!< Time the request was created.
account_flags creation_flags; //!< Generated locally?
char reserved[3];
};
static_assert(sizeof(request_info) == 64 + 32 + 8 + (4 * 2), "padding in request_info");
}
namespace v1
@@ -302,8 +328,11 @@ namespace db
constexpr const lmdb::basic_table<unsigned, block_pow> pows{
"pow_by_id", (MDB_CREATE | MDB_DUPSORT), MONERO_SORT_BY(block_pow, id)
};
constexpr const lmdb::basic_table<account_status, v0::account> accounts_v0{
"accounts_by_status,id", MDB_DUPSORT, MONERO_SORT_BY(v0::account, id)
};
constexpr const lmdb::basic_table<account_status, account> accounts{
"accounts_by_status,id", (MDB_CREATE | MDB_DUPSORT), MONERO_SORT_BY(account, id)
"accounts_v1_by_status,id", (MDB_CREATE | MDB_DUPSORT), MONERO_SORT_BY(account, id)
};
constexpr const lmdb::basic_table<unsigned, account_by_address> accounts_by_address(
"accounts_by_address", (MDB_CREATE | MDB_DUPSORT), MONERO_COMPARE(account_by_address, address.view_public)
@@ -329,8 +358,11 @@ namespace db
constexpr const lmdb::basic_table<output_id, db::key_image> images{
"key_images_by_output_id,image", (MDB_CREATE | MDB_DUPSORT), MONERO_COMPARE(db::key_image, value)
};
constexpr const lmdb::basic_table<request, v0::request_info> requests_v0{
"requests_by_type,address", MDB_DUPSORT, MONERO_COMPARE(v0::request_info, address.spend_public)
};
constexpr const lmdb::basic_table<request, request_info> requests{
"requests_by_type,address", (MDB_CREATE | MDB_DUPSORT), MONERO_COMPARE(request_info, address.spend_public)
"requests_v1_by_type,address", (MDB_CREATE | MDB_DUPSORT), MONERO_COMPARE(request_info, address.spend_public)
};
constexpr const lmdb::msgpack_table<webhook_key, webhook_dupsort, webhook_data> webhooks{
"webhooks_by_account_id,payment_id", (MDB_CREATE | MDB_DUPSORT), &lmdb::less<db::webhook_dupsort>
@@ -660,6 +692,14 @@ namespace db
} while (err == 0);
return log_lmdb_error(err, __LINE__, __FILE__);
}
template<typename T>
T add_and_clamp(T start, T count) noexcept
{
if (std::numeric_limits<T>::max() - count < start)
return std::numeric_limits<T>::max();
return start + count;
}
} // anonymous
struct storage_internal : lws_lmdb::database
@@ -721,6 +761,18 @@ namespace db
else if (v0_spends != lmdb::error(MDB_NOTFOUND))
MONERO_THROW(v0_spends.error(), "Error opening old spends table");
const auto v0_accounts = accounts_v0.open(*txn);
if (v0_accounts)
MONERO_UNWRAP(convert_table<v0::account, account>(*txn, *v0_accounts, tables.accounts));
else if (v0_accounts != lmdb::error(MDB_NOTFOUND))
MONERO_THROW(v0_accounts.error(), "Error opening old accounts table");
const auto v0_requests = requests_v0.open(*txn);
if (v0_requests)
MONERO_UNWRAP(convert_table<v0::request_info, request_info>(*txn, *v0_accounts, tables.requests));
else if (v0_requests != lmdb::error(MDB_NOTFOUND))
MONERO_THROW(v0_requests.error(), "Error open old requests table");
check_blockchain(*txn, tables.blocks);
check_pow(*txn, tables.pows);
MONERO_UNWRAP(this->commit(std::move(txn)));
@@ -2116,9 +2168,301 @@ namespace db
namespace
{
expect<std::vector<subaddress_dict>> do_upsert(MDB_cursor& ranges_cur, MDB_cursor& indexes_cur, const account_id id, const account_address& address, const crypto::secret_key& view_key, std::vector<subaddress_dict> subaddrs, const std::uint32_t max_subaddr)
{
std::size_t subaddr_count = 0;
std::vector<subaddress_dict> out{};
index_ranges new_dict{};
const auto add_out = [&out] (major_index major, index_range minor)
{
if (out.empty() || out.back().first != major)
out.emplace_back(major, index_ranges{std::vector<index_range>{minor}});
else
out.back().second.get_container().push_back(minor);
};
const auto check_max_range = [&subaddr_count, max_subaddr] (const index_range& range) -> bool
{
const auto more = std::uint32_t(range[1]) - std::uint32_t(range[0]);
if (max_subaddr - subaddr_count <= more)
return false;
subaddr_count += more + 1;
return true;
};
const auto check_max_ranges = [&check_max_range] (const index_ranges& ranges) -> bool
{
for (const auto& range : ranges.get_container())
{
if (!check_max_range(range))
return false;
}
return true;
};
MDB_val key = lmdb::to_val(id);
MDB_val value{};
int err = mdb_cursor_get(&indexes_cur, &key, &value, MDB_SET);
if (err)
{
if (err != MDB_NOTFOUND)
return log_lmdb_error(err, __LINE__, __FILE__);
}
else
{
MLWS_LMDB_CHECK(mdb_cursor_count(&indexes_cur, &subaddr_count));
if (max_subaddr < subaddr_count)
return {error::max_subaddresses};
}
for (auto& major_entry : subaddrs)
{
new_dict.get_container().clear();
if (!check_subaddress_dict(major_entry))
{
MERROR("Invalid subaddress_dict given to storage::upsert_subaddrs");
return {wire::error::schema::array};
}
value = lmdb::to_val(major_entry.first);
err = mdb_cursor_get(&ranges_cur, &key, &value, MDB_GET_BOTH);
if (err)
{
if (err != MDB_NOTFOUND)
return log_lmdb_error(err, __LINE__, __FILE__);
if (!check_max_ranges(major_entry.second))
return {error::max_subaddresses};
out.push_back(major_entry);
new_dict = std::move(major_entry.second);
}
else // merge new minor index ranges with old
{
auto old_dict = subaddress_ranges.get_value(value);
if (!old_dict)
return old_dict.error();
MLWS_LMDB_CHECK(mdb_cursor_del(&ranges_cur, 0)); // updated at end
auto& old_range = old_dict->second.get_container();
const auto& new_range = major_entry.second.get_container();
auto old_loc = old_range.begin();
auto new_loc = new_range.begin();
for ( ; old_loc != old_range.end() && new_loc != new_range.end(); )
{
if (std::uint64_t(new_loc->at(1)) + 1 < std::uint32_t(old_loc->at(0)))
{ // new has no overlap with existing
if (!check_max_range(*new_loc))
return {error::max_subaddresses};
new_dict.get_container().push_back(*new_loc);
add_out(major_entry.first, *new_loc);
++new_loc;
}
else if (std::uint64_t(old_loc->at(1)) + 1 < std::uint32_t(new_loc->at(0)))
{ // existing has no overlap with new
new_dict.get_container().push_back(*old_loc);
++old_loc;
}
else if (old_loc->at(0) <= new_loc->at(0) && new_loc->at(1) <= old_loc->at(1))
{ // new is completely within existing
++new_loc;
}
else // new overlap at beginning, end, or both
{
if (new_loc->at(0) < old_loc->at(0))
{ // overlap at beginning
const index_range new_range{new_loc->at(0), minor_index(std::uint32_t(old_loc->at(0)) - 1)};
if (!check_max_range(new_range))
return {error::max_subaddresses};
add_out(major_entry.first, new_range);
old_loc->at(0) = new_loc->at(0);
}
if (old_loc->at(1) < new_loc->at(1))
{ // overlap at end
const index_range new_range{minor_index(std::uint32_t(old_loc->at(1)) + 1), new_loc->at(1)};
if (!check_max_range(new_range))
return {error::max_subaddresses};
add_out(major_entry.first, new_range);
old_loc->at(1) = new_loc->at(1);
}
++new_loc;
}
}
std::copy(old_loc, old_range.end(), std::back_inserter(new_dict.get_container()));
for ( ; new_loc != new_range.end(); ++new_loc)
{
if (!check_max_range(*new_loc))
return {error::max_subaddresses};
new_dict.get_container().push_back(*new_loc);
add_out(major_entry.first, *new_loc);
}
}
for (const auto& new_indexes : new_dict.get_container())
{
for (std::uint64_t minor : boost::counting_range(std::uint64_t(new_indexes[0]), std::uint64_t(new_indexes[1]) + 1))
{
subaddress_map new_value{};
new_value.index = address_index{major_entry.first, minor_index(minor)};
new_value.subaddress = new_value.index.get_spend_public(address, view_key);
key = lmdb::to_val(id);
value = lmdb::to_val(new_value);
const int err = mdb_cursor_put(&indexes_cur, &key, &value, MDB_NODUPDATA);
if (err && err != MDB_KEYEXIST)
return log_lmdb_error(err, __LINE__, __FILE__);
}
}
const expect<epee::byte_slice> value_bytes =
subaddress_ranges.make_value(major_entry.first, new_dict);
if (!value_bytes)
return value_bytes.error();
key = lmdb::to_val(id);
value = MDB_val{value_bytes->size(), const_cast<void*>(static_cast<const void*>(value_bytes->data()))};
MLWS_LMDB_CHECK(mdb_cursor_put(&ranges_cur, &key, &value, MDB_NODUPDATA));
}
return {std::move(out)};
}
expect<void> do_lookahead(MDB_cursor& outputs_cur, MDB_cursor& ranges_cur, MDB_cursor& indexes_cur, const account_id id, const account_address& address, const crypto::secret_key& view_key, const address_index& lookahead, const std::uint32_t max_subaddresses)
{
const auto major = to_uint(lookahead.maj_i);
const auto minor = to_uint(lookahead.min_i);
if (!major || !minor)
return success();
// Quick fail check
if (std::numeric_limits<std::uint32_t>::max() < major / minor)
return {error::max_subaddresses};
if (max_subaddresses < major * minor)
return {error::max_subaddresses};
MDB_val key = lmdb::to_val(id);
MDB_val value{};
std::vector<subaddress_dict> initial{};
initial.resize(major);
/* Cycle through all received subaddresses, record highest minor index
within each major index. */
{
int err = mdb_cursor_get(&outputs_cur, &key, &value, MDB_SET);
if (err && err != MDB_NOTFOUND)
return log_lmdb_error(err, __LINE__, __FILE__);
if (!err)
{
err = mdb_cursor_get(&outputs_cur, &key, &value, MDB_FIRST_DUP);
for (;;)
{
if (err)
{
if (err == MDB_NOTFOUND)
break;
return log_lmdb_error(err, __LINE__, __FILE__);
}
const auto receipient = outputs.get_value<MONERO_FIELD(output, recipient)>(value);
if (!receipient)
return receipient.error();
if (!receipient->is_zero())
{
const auto index = to_uint(receipient->maj_i);
const auto needed = add_and_clamp(index, major);
const auto this_minor = add_and_clamp(minor - 1, to_uint(receipient->min_i));
// Quick Sanity Check before vector expansion
if (std::numeric_limits<std::uint32_t>::max() < needed / minor)
return {error::max_subaddresses};
if (max_subaddresses < needed * minor)
return {error::max_subaddresses};
static_assert(
std::numeric_limits<decltype(needed)>::max() <=
std::numeric_limits<std::size_t>::max()
);
initial.resize(std::max(initial.size(), std::size_t(needed)));
auto& elem = initial.at(index);
if (!elem.second.get_container().empty())
{
auto& current = std::get<1>(elem.second.get_container().at(0));
current = std::max(current, minor_index(this_minor));
}
else
elem = {major_index(major), index_ranges{{index_range{minor_index(0), minor_index(this_minor)}}}};
err = mdb_cursor_get(&outputs_cur, &key, &value, MDB_NEXT_DUP);
}
}
}
}
// fill in every gap with a default {0, minor} subaddress range
{
const index_ranges default_{{index_range{minor_index(0), minor_index(minor - 1)}}};
std::size_t i = -1;
std::uint32_t total_subs = 0;
for (auto& elem : initial)
{
++i;
elem.first = major_index(i);
auto this_minor = minor;
if (elem.second.get_container().empty())
elem.second.get_container() = default_.get_container();
else
this_minor = to_uint(std::get<1>(elem.second.get_container().at(0)));
if (max_subaddresses - total_subs < this_minor)
return {error::max_subaddresses};
total_subs += this_minor;
}
}
const auto upserted = do_upsert(ranges_cur, indexes_cur, id, address, view_key, std::move(initial), max_subaddresses);
if (!upserted)
return upserted.error();
return success();
}
expect<void> do_lookahead(account& user, MDB_cursor& outputs_cur, MDB_cursor& ranges_cur, MDB_cursor& indexes_cur, const address_index& lookahead, const std::uint32_t max_subaddresses)
{
crypto::secret_key key;
static_assert(sizeof(key) == sizeof(user.key));
std::memcpy(std::addressof(unwrap(unwrap(key))), std::addressof(user.key), sizeof(key));
const expect<void> attempt =
do_lookahead(outputs_cur, ranges_cur, indexes_cur, user.id, user.address, key, lookahead, max_subaddresses);
if (attempt == error::max_subaddresses)
{
user.lookahead = lookahead;
user.lookahead_fail = std::max(block_id(1), user.start_height);
return success();
}
else if (attempt)
user.lookahead = lookahead;
return attempt;
}
//! \return Success, even if `address` was not found (designed for
expect<void>
change_height(MDB_cursor& accounts_cur, MDB_cursor& accounts_ba_cur, MDB_cursor& accounts_bh_cur, MDB_cursor& outputs_cur, MDB_cursor& spends_cur, MDB_cursor& images_cur, block_id height, account_address const& address)
change_height(
MDB_cursor& accounts_cur,
MDB_cursor& accounts_ba_cur,
MDB_cursor& accounts_bh_cur,
MDB_cursor& outputs_cur,
MDB_cursor& spends_cur,
MDB_cursor& images_cur,
MDB_cursor* ranges_cur,
MDB_cursor* indexes_cur,
block_id height,
account_address const& address,
address_index lookahead = {},
std::uint32_t max_subaddresses = 0)
{
MDB_val key = lmdb::to_val(by_address_version);
MDB_val value = lmdb::to_val(address);
@@ -2146,6 +2490,15 @@ namespace db
const block_id current_height = user->scan_height;
user->scan_height = std::min(height, user->scan_height);
user->start_height = std::min(height, user->start_height);
if (height <= user->lookahead_fail)
user->lookahead_fail = block_id(0);
// rollback before lookahead
MONERO_CHECK(rollback_outputs(user->id, height, outputs_cur));
MONERO_CHECK(rollback_spends(user->id, height, spends_cur, images_cur));
if (!lookahead.is_zero() && ranges_cur && indexes_cur)
MONERO_CHECK(do_lookahead(*user, outputs_cur, *ranges_cur, *indexes_cur, lookahead, max_subaddresses));
key = lmdb::to_val(lookup->status);
value = lmdb::to_val(*user);
@@ -2165,9 +2518,6 @@ namespace db
mdb_cursor_put(&accounts_bh_cur, &key, &value, MDB_NODUPDATA)
);
MONERO_CHECK(rollback_outputs(user->id, height, outputs_cur));
MONERO_CHECK(rollback_spends(user->id, height, spends_cur, images_cur));
return success();
}
}
@@ -2216,7 +2566,7 @@ namespace db
for (account_address const& address : addresses)
{
const expect<void> changed = change_height(
*accounts_cur, *accounts_ba_cur, *accounts_bh_cur, *outputs_cur, *spends_cur, *images_cur, height, address
*accounts_cur, *accounts_ba_cur, *accounts_bh_cur, *outputs_cur, *spends_cur, *images_cur, nullptr, nullptr, height, address
);
if (changed)
updated.push_back(address);
@@ -2227,14 +2577,14 @@ namespace db
});
}
expect<std::vector<webhook_new_account>> storage::creation_request(account_address const& address, crypto::secret_key const& key, account_flags flags) noexcept
expect<std::vector<webhook_new_account>> storage::creation_request(account_address const& address, crypto::secret_key const& key, account_flags flags, address_index lookahead) noexcept
{
MONERO_PRECOND(db != nullptr);
if (!db->create_queue_max)
return {lws::error::create_queue_max};
return db->try_write([this, &address, &key, flags] (MDB_txn& txn) -> expect<std::vector<webhook_new_account>>
return db->try_write([this, &address, &key, flags, lookahead] (MDB_txn& txn) -> expect<std::vector<webhook_new_account>>
{
const expect<db::account_time> current_time = get_account_time();
if (!current_time)
@@ -2293,6 +2643,7 @@ namespace db
info.creation = *current_time;
info.start_height = *height;
info.creation_flags = flags;
info.lookahead = lookahead;
keyv = lmdb::to_val(req);
value = lmdb::to_val(info);
@@ -2324,10 +2675,10 @@ namespace db
});
}
expect<void> storage::import_request(account_address const& address, block_id height) noexcept
expect<void> storage::import_request(account_address const& address, block_id height, address_index lookahead) noexcept
{
MONERO_PRECOND(db != nullptr);
return db->try_write([this, &address, height] (MDB_txn& txn) -> expect<void>
return db->try_write([this, &address, height, lookahead] (MDB_txn& txn) -> expect<void>
{
const expect<db::account_time> current_time = get_account_time();
if (!current_time)
@@ -2351,6 +2702,7 @@ namespace db
request_info info{};
info.address = address;
info.start_height = height;
info.lookahead = lookahead;
const request req = request::import_scan;
key = lmdb::to_val(req);
@@ -2366,10 +2718,60 @@ namespace db
});
}
expect<void> storage::shrink_lookahead(account_address const& address, const address_index& lookahead) noexcept
{
// shrink is simple because the upsert logic never has to be run
MONERO_PRECOND(db != nullptr);
return db->try_write([this, &address, &lookahead] (MDB_txn& txn) -> expect<void>
{
cursor::accounts accounts_cur;
cursor::accounts_by_address accounts_ba_cur;
MONERO_CHECK(check_cursor(txn, this->db->tables.accounts, accounts_cur));
MONERO_CHECK(check_cursor(txn, this->db->tables.accounts_ba, accounts_ba_cur));
MDB_val key = lmdb::to_val(by_address_version);
MDB_val value = lmdb::to_val(address);
const int err = mdb_cursor_get(accounts_ba_cur.get(), &key, &value, MDB_GET_BOTH);
if (err == MDB_NOTFOUND)
return {lws::error::account_not_found};
if (err)
return log_lmdb_error(err, __LINE__, __FILE__);
const expect<account_lookup> lookup =
accounts_by_address.get_value<MONERO_FIELD(account_by_address, lookup)>(value);
if (!lookup)
return lookup.error();
key = lmdb::to_val(lookup->status);
value = lmdb::to_val(lookup->id);
MLWS_LMDB_CHECK(
mdb_cursor_get(accounts_cur.get(), &key, &value, MDB_GET_BOTH)
);
expect<account> user = accounts.get_value<account>(value);
if (!user)
return user.error();
if (user->lookahead_fail != block_id(0) || user->lookahead.maj_i < lookahead.maj_i || user->lookahead.min_i < lookahead.min_i)
return {error::max_subaddresses};
user->lookahead = lookahead;
value = lmdb::to_val(*user);
key = lmdb::to_val(lookup->status);
MLWS_LMDB_CHECK(
mdb_cursor_put(accounts_cur.get(), &key, &value, MDB_CURRENT)
);
return success();
});
}
namespace
{
expect<std::vector<account_address>>
create_accounts(MDB_txn& txn, storage_internal::tables_ const& tables, epee::span<const account_address> addresses)
create_accounts(MDB_txn& txn, storage_internal::tables_ const& tables, epee::span<const account_address> addresses, const std::uint32_t max_subaddresses)
{
std::vector<account_address> stored{};
stored.reserve(addresses.size());
@@ -2382,11 +2784,17 @@ namespace db
cursor::accounts_by_address accounts_ba_cur;
cursor::accounts_by_height accounts_bh_cur;
cursor::requests requests_cur;
cursor::subaddress_ranges ranges_cur;
cursor::subaddress_indexes indexes_cur;
cursor::outputs outputs_cur;
MONERO_CHECK(check_cursor(txn, tables.accounts, accounts_cur));
MONERO_CHECK(check_cursor(txn, tables.accounts_ba, accounts_ba_cur));
MONERO_CHECK(check_cursor(txn, tables.accounts_bh, accounts_bh_cur));
MONERO_CHECK(check_cursor(txn, tables.requests, requests_cur));
MONERO_CHECK(check_cursor(txn, tables.subaddress_ranges, ranges_cur));
MONERO_CHECK(check_cursor(txn, tables.subaddress_indexes, indexes_cur));
MONERO_CHECK(check_cursor(txn, tables.outputs, outputs_cur));
expect<account_id> last_id = find_last_id(*accounts_cur);
if (!last_id)
@@ -2422,6 +2830,8 @@ namespace db
user.access = *current_time;
user.creation = info->creation;
user.flags = info->creation_flags;
if (!info->lookahead.is_zero())
MONERO_CHECK(do_lookahead(user, *outputs_cur, *ranges_cur, *indexes_cur, info->lookahead, max_subaddresses));
const expect<void> added =
do_add_account(*accounts_cur, *accounts_ba_cur, *accounts_bh_cur, user);
@@ -2440,7 +2850,7 @@ namespace db
}
expect<std::vector<account_address>>
import_accounts(MDB_txn& txn, storage_internal::tables_ const& tables, epee::span<const account_address> addresses)
import_accounts(MDB_txn& txn, storage_internal::tables_ const& tables, epee::span<const account_address> addresses, const std::uint32_t max_subaddresses)
{
std::vector<account_address> updated{};
updated.reserve(addresses.size());
@@ -2452,6 +2862,8 @@ namespace db
cursor::outputs outputs_cur;
cursor::spends spends_cur;
cursor::images images_cur;
cursor::subaddress_ranges ranges_cur;
cursor::subaddress_indexes indexes_cur;
MONERO_CHECK(check_cursor(txn, tables.accounts, accounts_cur));
MONERO_CHECK(check_cursor(txn, tables.accounts_ba, accounts_ba_cur));
@@ -2460,6 +2872,8 @@ namespace db
MONERO_CHECK(check_cursor(txn, tables.outputs, outputs_cur));
MONERO_CHECK(check_cursor(txn, tables.spends, spends_cur));
MONERO_CHECK(check_cursor(txn, tables.images, images_cur));
MONERO_CHECK(check_cursor(txn, tables.subaddress_ranges, ranges_cur));
MONERO_CHECK(check_cursor(txn, tables.subaddress_indexes, indexes_cur));
const request req = request::import_scan;
for (account_address const& address : addresses)
@@ -2477,9 +2891,13 @@ namespace db
MLWS_LMDB_CHECK(mdb_cursor_del(requests_cur.get(), 0));
if (!new_height)
return new_height.error();
const expect<address_index> lookahead =
requests.get_value<MONERO_FIELD(request_info, lookahead)>(value);
if (!lookahead)
return lookahead.error();
const expect<void> changed = change_height(
*accounts_cur, *accounts_ba_cur, *accounts_bh_cur, *outputs_cur, *spends_cur, *images_cur, *new_height, address
*accounts_cur, *accounts_ba_cur, *accounts_bh_cur, *outputs_cur, *spends_cur, *images_cur, ranges_cur.get(), indexes_cur.get(), *new_height, address, *lookahead, max_subaddresses
);
if (changed)
updated.push_back(address);
@@ -2491,20 +2909,20 @@ namespace db
} // anonymous
expect<std::vector<account_address>>
storage::accept_requests(request req, epee::span<const account_address> addresses)
storage::accept_requests(request req, epee::span<const account_address> addresses, const std::uint32_t max_subaddresses)
{
if (addresses.empty())
return std::vector<account_address>{};
MONERO_PRECOND(db != nullptr);
return db->try_write([this, req, addresses] (MDB_txn& txn) -> expect<std::vector<account_address>>
return db->try_write([this, req, addresses, max_subaddresses] (MDB_txn& txn) -> expect<std::vector<account_address>>
{
switch (req)
{
case request::create:
return create_accounts(txn, this->db->tables, addresses);
return create_accounts(txn, this->db->tables, addresses, max_subaddresses);
case request::import_scan:
return import_accounts(txn, this->db->tables, addresses);
return import_accounts(txn, this->db->tables, addresses, max_subaddresses);
default:
break;
}
@@ -2920,167 +3338,125 @@ namespace db
return db->try_write([this, id, &address, &view_key, &subaddrs, max_subaddr] (MDB_txn& txn) -> expect<std::vector<subaddress_dict>>
{
std::size_t subaddr_count = 0;
std::vector<subaddress_dict> out{};
index_ranges new_dict{};
const auto add_out = [&out] (major_index major, index_range minor)
{
if (out.empty() || out.back().first != major)
out.emplace_back(major, index_ranges{std::vector<index_range>{minor}});
else
out.back().second.get_container().push_back(minor);
};
const auto check_max_range = [&subaddr_count, max_subaddr] (const index_range& range) -> bool
{
const auto more = std::uint32_t(range[1]) - std::uint32_t(range[0]);
if (max_subaddr - subaddr_count <= more)
return false;
subaddr_count += more + 1;
return true;
};
const auto check_max_ranges = [&check_max_range] (const index_ranges& ranges) -> bool
{
for (const auto& range : ranges.get_container())
{
if (!check_max_range(range))
return false;
}
return true;
};
cursor::subaddress_ranges ranges_cur;
cursor::subaddress_indexes indexes_cur;
MONERO_CHECK(check_cursor(txn, this->db->tables.subaddress_ranges, ranges_cur));
MONERO_CHECK(check_cursor(txn, this->db->tables.subaddress_indexes, indexes_cur));
MDB_val key = lmdb::to_val(id);
MDB_val value{};
int err = mdb_cursor_get(indexes_cur.get(), &key, &value, MDB_SET);
return do_upsert(*ranges_cur, *indexes_cur, id, address, view_key, subaddrs, max_subaddr);
});
}
expect<std::int64_t> storage::update_lookahead(const account_address& address, block_id height, address_index match, std::uint32_t max_subaddresses)
{
MONERO_PRECOND(db != nullptr);
return db->try_write([this, &address, height, match, max_subaddresses] (MDB_txn& txn) -> expect<std::int64_t>
{
cursor::accounts accounts_cur;
cursor::accounts_by_address accounts_ba_cur;
cursor::subaddress_ranges ranges_cur;
cursor::subaddress_indexes indexes_cur;
MONERO_CHECK(check_cursor(txn, this->db->tables.accounts, accounts_cur));
MONERO_CHECK(check_cursor(txn, this->db->tables.accounts_ba, accounts_ba_cur));
MONERO_CHECK(check_cursor(txn, this->db->tables.subaddress_ranges, ranges_cur));
MONERO_CHECK(check_cursor(txn, this->db->tables.subaddress_indexes, indexes_cur));
MDB_val key = lmdb::to_val(by_address_version);
MDB_val value = lmdb::to_val(address);
const int err = mdb_cursor_get(accounts_ba_cur.get(), &key, &value, MDB_GET_BOTH);
if (err == MDB_NOTFOUND)
return {lws::error::account_not_found};
if (err)
return log_lmdb_error(err, __LINE__, __FILE__);
const expect<account_lookup> lookup =
accounts_by_address.get_value<MONERO_FIELD(account_by_address, lookup)>(value);
if (!lookup)
return lookup.error();
key = lmdb::to_val(lookup->status);
value = lmdb::to_val(lookup->id);
MLWS_LMDB_CHECK(
mdb_cursor_get(accounts_cur.get(), &key, &value, MDB_GET_BOTH)
);
expect<account> user = accounts.get_value<account>(value);
if (!user)
return user.error();
const auto major = to_uint(user->lookahead.maj_i);
const auto minor = to_uint(user->lookahead.min_i);
if (!major || !minor)
return 0;
expect<std::vector<subaddress_dict>> upserted{error::max_subaddresses};
if (major / minor <= std::numeric_limits<decltype(major)>::max())
{
if (err != MDB_NOTFOUND)
return log_lmdb_error(err, __LINE__, __FILE__);
}
else
{
MLWS_LMDB_CHECK(mdb_cursor_count(indexes_cur.get(), &subaddr_count));
if (max_subaddr < subaddr_count)
return {error::max_subaddresses};
}
for (auto& major_entry : subaddrs)
{
new_dict.get_container().clear();
if (!check_subaddress_dict(major_entry))
{
MERROR("Invalid subaddress_dict given to storage::upsert_subaddrs");
return {wire::error::schema::array};
}
value = lmdb::to_val(major_entry.first);
err = mdb_cursor_get(ranges_cur.get(), &key, &value, MDB_GET_BOTH);
if (err)
{
if (err != MDB_NOTFOUND)
return log_lmdb_error(err, __LINE__, __FILE__);
if (!check_max_ranges(major_entry.second))
return {error::max_subaddresses};
out.push_back(major_entry);
new_dict = std::move(major_entry.second);
}
else // merge new minor index ranges with old
{
auto old_dict = subaddress_ranges.get_value(value);
if (!old_dict)
return old_dict.error();
mdb_cursor_del(ranges_cur.get(), 0); // updated at end
auto& old_range = old_dict->second.get_container();
const auto& new_range = major_entry.second.get_container();
auto old_loc = old_range.begin();
auto new_loc = new_range.begin();
for ( ; old_loc != old_range.end() && new_loc != new_range.end(); )
if (major * minor <= max_subaddresses)
{
std::vector<db::subaddress_dict> upsertions;
const auto fresh = to_uint(match.maj_i);
const auto end = add_and_clamp(fresh, major);
for (std::uint64_t i = 0; i < end; ++i)
{
if (std::uint64_t(new_loc->at(1)) + 1 < std::uint32_t(old_loc->at(0)))
{ // new has no overlap with existing
if (!check_max_range(*new_loc))
return {error::max_subaddresses};
new_dict.get_container().push_back(*new_loc);
add_out(major_entry.first, *new_loc);
++new_loc;
}
else if (std::uint64_t(old_loc->at(1)) + 1 < std::uint32_t(new_loc->at(0)))
{ // existing has no overlap with new
new_dict.get_container().push_back(*old_loc);
++old_loc;
}
else if (old_loc->at(0) <= new_loc->at(0) && new_loc->at(1) <= old_loc->at(1))
{ // new is completely within existing
++new_loc;
}
else // new overlap at beginning, end, or both
if (i == fresh)
{
if (new_loc->at(0) < old_loc->at(0))
{ // overlap at beginning
const index_range new_range{new_loc->at(0), minor_index(std::uint32_t(old_loc->at(0)) - 1)};
if (!check_max_range(new_range))
return {error::max_subaddresses};
add_out(major_entry.first, new_range);
old_loc->at(0) = new_loc->at(0);
}
if (old_loc->at(1) < new_loc->at(1))
{ // overlap at end
const index_range new_range{minor_index(std::uint32_t(old_loc->at(1)) + 1), new_loc->at(1)};
if (!check_max_range(new_range))
return {error::max_subaddresses};
add_out(major_entry.first, new_range);
old_loc->at(1) = new_loc->at(1);
}
++new_loc;
upsertions.emplace_back(
db::major_index(i),
db::index_ranges{{
db::index_range{
db::minor_index(0),
db::minor_index(add_and_clamp(to_uint(match.min_i), minor - 1))
}
}}
);
}
else
{
upsertions.emplace_back(
db::major_index(i),
db::index_ranges{{
db::index_range{db::minor_index(0), db::minor_index(minor - 1)}
}}
);
}
}
std::copy(old_loc, old_range.end(), std::back_inserter(new_dict.get_container()));
for ( ; new_loc != new_range.end(); ++new_loc)
{
if (!check_max_range(*new_loc))
return {error::max_subaddresses};
new_dict.get_container().push_back(*new_loc);
add_out(major_entry.first, *new_loc);
}
crypto::secret_key viewkey;
static_assert(sizeof(viewkey) == sizeof(user->key));
std::memcpy(std::addressof(unwrap(unwrap(viewkey))), std::addressof(user->key), sizeof(viewkey));
upserted =
do_upsert(*ranges_cur, *indexes_cur, user->id, address, viewkey, std::move(upsertions), max_subaddresses);
}
for (const auto& new_indexes : new_dict.get_container())
{
for (std::uint64_t minor : boost::counting_range(std::uint64_t(new_indexes[0]), std::uint64_t(new_indexes[1]) + 1))
{
subaddress_map new_value{};
new_value.index = address_index{major_entry.first, minor_index(minor)};
new_value.subaddress = new_value.index.get_spend_public(address, view_key);
key = lmdb::to_val(id);
value = lmdb::to_val(new_value);
const int err = mdb_cursor_put(indexes_cur.get(), &key, &value, MDB_NODUPDATA);
if (err && err != MDB_KEYEXIST)
return log_lmdb_error(err, __LINE__, __FILE__);
}
}
const expect<epee::byte_slice> value_bytes =
subaddress_ranges.make_value(major_entry.first, new_dict);
if (!value_bytes)
return value_bytes.error();
key = lmdb::to_val(id);
value = MDB_val{value_bytes->size(), const_cast<void*>(static_cast<const void*>(value_bytes->data()))};
MLWS_LMDB_CHECK(mdb_cursor_put(ranges_cur.get(), &key, &value, MDB_NODUPDATA));
}
return {std::move(out)};
if (!upserted)
{
if (upserted != error::max_subaddresses)
return upserted.error();
if (user->lookahead_fail == block_id(0))
user->lookahead_fail = std::max(block_id(1), height);
else
user->lookahead_fail = std::max(block_id(1), std::min(user->lookahead_fail, height));
key = lmdb::to_val(lookup->status);
value = lmdb::to_val(*user);
MLWS_LMDB_CHECK(
mdb_cursor_put(accounts_cur.get(), &key, &value, MDB_CURRENT)
);
return -1;
}
static_assert(
std::numeric_limits<std::uint32_t>::max() <=
std::numeric_limits<std::size_t>::max()
);
return std::min(std::size_t(std::numeric_limits<std::uint32_t>::max()), upserted->size());
});
}

14
src/db/storage.h
View File

@@ -255,17 +255,20 @@ namespace db
rescan(block_id height, epee::span<const account_address> addresses);
//! Add an account for later approval. For use with the login endpoint.
expect<std::vector<webhook_new_account>> creation_request(account_address const& address, crypto::secret_key const& key, account_flags flags) noexcept;
expect<std::vector<webhook_new_account>> creation_request(account_address const& address, crypto::secret_key const& key, account_flags flags, address_index lookahead) noexcept;
/*!
Request lock height of an existing account. No effect if the `start_height`
is already older.
*/
expect<void> import_request(account_address const& address, block_id height) noexcept;
expect<void> import_request(account_address const& address, block_id height, address_index lookahead) noexcept;
//! Shrink but not expand lookahead for `address`.
expect<void> shrink_lookahead(account_address const& address, const address_index& lookahead) noexcept;
//! Accept requests by `addresses` of type `req`. \return Accepted addresses.
expect<std::vector<account_address>>
accept_requests(request req, epee::span<const account_address> addresses);
accept_requests(request req, epee::span<const account_address> addresses, std::uint32_t max_subaddresses);
//! Reject requests by `addresses` of type `req`. \return Rejected addresses.
expect<std::vector<account_address>>
@@ -311,6 +314,11 @@ namespace db
expect<std::vector<subaddress_dict>>
upsert_subaddresses(account_id id, const account_address& address, const crypto::secret_key& view_key, std::vector<subaddress_dict> subaddrs, std::uint32_t max_subaddresses);
/*! Update lookahead where `match` was a matching subaddress on-chain.
\return The number of new subaddresses added via lookahead, or -1 if
`max_subaddresses was reached. */
expect<std::int64_t> update_lookahead(const account_address& address, block_id height, address_index match, std::uint32_t max_subaddresses);
/*!
Add webhook to be tracked in the database. The webhook will "call"
the specified URL with JSON/msgpack information when the event occurs.

95
src/rest_server.cpp
View File

@@ -97,6 +97,7 @@ namespace lws
const epee::net_utils::ssl_verification_t webhook_verify;
const bool disable_admin_auth;
const bool auto_accept_creation;
const bool auto_accept_import;
};
struct rest_server_data
@@ -265,6 +266,19 @@ namespace lws
return {std::make_pair(user->second, std::move(*reader))};
}
bool check_lookahead(connection_data& data, const db::address_index lookahead)
{
const auto minor = to_uint(lookahead.min_i);
if (minor)
{
const auto major = to_uint(lookahead.maj_i);
if (std::numeric_limits<std::uint32_t>::max() < major / minor)
return false;
return major * minor <= data.global->options.max_subaddresses;
}
return true;
}
//! For endpoints that _sometimes_ generate async responses
expect<epee::byte_slice> async_ready() noexcept
{ return epee::byte_slice{}; }
@@ -521,6 +535,8 @@ namespace lws
resp.scanned_height = std::uint64_t(user->first.scan_height);
resp.scanned_block_height = resp.scanned_height;
resp.start_height = std::uint64_t(user->first.start_height);
resp.lookahead_fail = to_uint(user->first.lookahead_fail);
resp.lookahead = user->first.lookahead;
std::vector<db::output::spend_meta_> metas{};
metas.reserve(outputs->count());
@@ -595,6 +611,8 @@ namespace lws
resp.start_height = std::uint64_t(user->first.start_height);
resp.blockchain_height = std::uint64_t(last->id);
resp.transaction_height = resp.blockchain_height;
resp.lookahead_fail = to_uint(user->first.lookahead_fail);
resp.lookahead = user->first.lookahead;
// merge input and output info into a single set of txes.
@@ -1115,6 +1133,7 @@ namespace lws
per_byte_fee,
rpc->fee_mask,
rpc::safe_uint64(received),
to_uint(user->first.lookahead_fail),
std::move(unspent),
rpc->fees,
std::move(req.creds.key)
@@ -1340,13 +1359,28 @@ namespace lws
{
bool new_request = false;
bool fulfilled = false;
db::address_index lookahead{};
{
auto user = open_account(req.creds, data.global->disk.clone());
if (!user)
return user.error();
data.passed_login = true;
if (user->first.start_height <= db::block_id(req.from_height))
if (!check_lookahead(data, req.lookahead))
return {lws::error::max_subaddresses};
const auto expanded_depth = [&req] (const auto& record)
{ return db::block_id(req.from_height) < record.start_height; };
const auto change_lookahead = [&req] (const auto& record)
{
return record.lookahead.maj_i != req.lookahead.maj_i ||
record.lookahead.min_i != req.lookahead.min_i;
};
lookahead = user->first.lookahead;
const bool lookahead_fail = user->first.lookahead_fail != db::block_id(0);
if (!expanded_depth(user->first) && !change_lookahead(user->first) && !lookahead_fail)
fulfilled = true;
else
{
@@ -1358,17 +1392,48 @@ namespace lws
if (info != lmdb::error(MDB_NOTFOUND))
return info.error();
new_request = true;
// Shrink immediately if possible
if (!lookahead_fail && req.lookahead.maj_i <= user->first.lookahead.maj_i && req.lookahead.min_i <= user->first.lookahead.min_i)
{
fulfilled = !expanded_depth(user->first);
new_request = !fulfilled;
// if not same
if (user->first.lookahead.maj_i != req.lookahead.maj_i && user->first.lookahead.min_i != req.lookahead.min_i)
{
MONERO_CHECK(data.global->disk.clone().shrink_lookahead(req.creds.address, req.lookahead));
lookahead = req.lookahead;
}
}
else
new_request = true;
}
}
} // close reader
if (new_request)
MONERO_CHECK(data.global->disk.clone().import_request(req.creds.address, db::block_id(req.from_height)));
{
auto disk = data.global->disk.clone();
MONERO_CHECK(disk.import_request(req.creds.address, db::block_id(req.from_height), req.lookahead));
if (data.global->options.auto_accept_import)
{
const auto accepted = disk.accept_requests(db::request::import_scan, {std::addressof(req.creds.address), 1}, data.global->options.max_subaddresses);
if (!accepted)
{
MERROR("Failed to import account " << db::address_string(req.creds.address) << ": " << accepted.error());
lookahead = {};
}
else
{
lookahead = req.lookahead;
fulfilled = true;
}
}
}
const char* status = new_request ?
"Accepted, waiting for approval" : (fulfilled ? "Approved" : "Waiting for Approval");
return response{rpc::safe_uint64(0), status, new_request, fulfilled};
return response{rpc::safe_uint64(0), status, lookahead, new_request, fulfilled};
}
};
@@ -1398,24 +1463,33 @@ namespace lws
// Do not count a request for account creation as login
data.passed_login = true;
return response{false, bool(account->second.flags & db::account_generated_locally)};
return response{false, bool(account->second.flags & db::account_generated_locally), account->second.lookahead};
}
else if (!req.create_account || account != lws::error::account_not_found)
return account.error();
}
if (!check_lookahead(data, req.lookahead))
return {lws::error::max_subaddresses};
const auto flags = req.generated_locally ? db::account_generated_locally : db::default_account;
const auto hooks = disk.creation_request(req.creds.address, req.creds.key, flags);
const auto hooks = disk.creation_request(req.creds.address, req.creds.key, flags, req.lookahead);
if (!hooks)
return hooks.error();
if (data.global->options.auto_accept_creation)
{
data.passed_login = true;
const auto accepted = disk.accept_requests(db::request::create, {std::addressof(req.creds.address), 1});
const auto accepted = disk.accept_requests(db::request::create, {std::addressof(req.creds.address), 1}, data.global->options.max_subaddresses);
if (!accepted)
{
MERROR("Failed to move account " << db::address_string(req.creds.address) << " to available state: " << accepted.error());
req.lookahead = {};
}
else
data.passed_login = true;
}
else
req.lookahead = {};
if (!hooks->empty())
{
@@ -1427,7 +1501,7 @@ namespace lws
);
}
return response{true, req.generated_locally};
return response{true, req.generated_locally, req.lookahead};
}
};
@@ -1788,6 +1862,7 @@ namespace lws
return error;
}
rpc::add_values(req.params, data.global->options); // add max_subaddresses
db::storage disk = data.global->disk.clone();
if (!data.global->options.disable_admin_auth)
{
@@ -2307,7 +2382,7 @@ namespace lws
}
rest_server::rest_server(epee::span<const std::string> addresses, std::vector<std::string> admin, db::storage disk, rpc::client client, configuration config)
: global_(std::make_unique<rest_server_data>(std::move(disk), std::move(client), runtime_options{config.max_subaddresses, config.webhook_verify, config.disable_admin_auth, config.auto_accept_creation})),
: global_(std::make_unique<rest_server_data>(std::move(disk), std::move(client), runtime_options{config.max_subaddresses, config.webhook_verify, config.disable_admin_auth, config.auto_accept_creation, config.auto_accept_import})),
ports_(),
workers_()
{

1
src/rest_server.h
View File

@@ -66,6 +66,7 @@ namespace lws
bool allow_external;
bool disable_admin_auth;
bool auto_accept_creation;
bool auto_accept_import;
};
explicit rest_server(epee::span<const std::string> addresses, std::vector<std::string> admin, db::storage disk, rpc::client client, configuration config);

3
src/rpc/admin.cpp
View File

@@ -164,6 +164,7 @@ namespace lws { namespace rpc
void read_bytes(wire::reader& source, address_requests& self)
{
read_addresses(source, self, WIRE_FIELD(type));
self.max_subaddresses = 0;
}
void read_bytes(wire::reader& source, modify_account_req& self)
{
@@ -205,7 +206,7 @@ namespace lws { namespace rpc
expect<void> accept_requests_::operator()(wire::writer& dest, db::storage disk, const request& req) const
{
return write_addresses(dest, disk.accept_requests(req.type, epee::to_span(req.addresses)));
return write_addresses(dest, disk.accept_requests(req.type, epee::to_span(req.addresses), req.max_subaddresses));
}
expect<void> add_account_::operator()(wire::writer& out, db::storage disk, const request& req) const

13
src/rpc/admin.h
View File

@@ -40,7 +40,7 @@
namespace lws
{
namespace rpc
{
{
struct add_account_req
{
db::account_address address;
@@ -52,6 +52,7 @@ namespace rpc
struct address_requests
{
std::vector<db::account_address> addresses;
std::uint32_t max_subaddresses;
db::request type;
};
void read_bytes(wire::reader&, address_requests&);
@@ -102,6 +103,16 @@ namespace rpc
void read_bytes(wire::reader&, webhook_delete_uuid_req&);
// Hack for passing max_subaddresses via cli option
template<typename T, typename U>
inline void add_values(const T&, const U&) noexcept
{}
template<typename T>
inline void add_values(address_requests& out, const T& src) noexcept
{ out.max_subaddresses = src.max_subaddresses; }
struct accept_requests_
{
using request = address_requests;

20
src/rpc/light_wallet.cpp
View File

@@ -266,8 +266,10 @@ namespace lws
WIRE_FIELD_COPY(start_height),
WIRE_FIELD_COPY(transaction_height),
WIRE_FIELD_COPY(blockchain_height),
WIRE_FIELD_DEFAULTED(lookahead_fail, unsigned(0)),
WIRE_FIELD(spent_outputs),
WIRE_OPTIONAL_FIELD(rates)
WIRE_OPTIONAL_FIELD(rates),
WIRE_FIELD_DEFAULTED(lookahead, db::address_index{})
);
}
@@ -318,6 +320,8 @@ namespace lws
WIRE_FIELD_COPY(start_height),
WIRE_FIELD_COPY(transaction_height),
WIRE_FIELD_COPY(blockchain_height),
WIRE_FIELD_DEFAULTED(lookahead_fail, unsigned(0)),
WIRE_FIELD_DEFAULTED(lookahead, db::address_index{}),
wire::optional_field("transactions", wire::array(boost::adaptors::index(self.transactions)))
);
}
@@ -359,6 +363,7 @@ namespace lws
WIRE_FIELD_COPY(per_byte_fee),
WIRE_FIELD_COPY(fee_mask),
WIRE_FIELD_COPY(amount),
WIRE_FIELD_DEFAULTED(lookahead_fail, unsigned(0)),
wire::optional_field("outputs", wire::array(boost::adaptors::transform(self.outputs, expand))),
WIRE_FIELD(fees)
);
@@ -400,7 +405,8 @@ namespace lws
wire::object(source,
wire::field("address", std::ref(address)),
wire::field("view_key", std::ref(unwrap(unwrap(self.creds.key)))),
WIRE_FIELD_DEFAULTED(from_height, unsigned(0))
WIRE_FIELD_DEFAULTED(from_height, unsigned(0)),
WIRE_FIELD_DEFAULTED(lookahead, db::address_index{})
);
convert_address(address, self.creds.address);
}
@@ -411,7 +417,8 @@ namespace lws
WIRE_FIELD_COPY(import_fee),
WIRE_FIELD(status),
WIRE_FIELD_COPY(new_request),
WIRE_FIELD_COPY(request_fulfilled)
WIRE_FIELD_COPY(request_fulfilled),
WIRE_FIELD_COPY(lookahead)
);
}
@@ -421,6 +428,7 @@ namespace lws
wire::object(source,
wire::field("address", std::ref(address)),
wire::field("view_key", std::ref(unwrap(unwrap(self.creds.key)))),
WIRE_FIELD_DEFAULTED(lookahead, db::address_index{}),
WIRE_FIELD(create_account),
WIRE_FIELD(generated_locally)
);
@@ -428,7 +436,11 @@ namespace lws
}
void rpc::write_bytes(wire::json_writer& dest, const login_response self)
{
wire::object(dest, WIRE_FIELD_COPY(new_address), WIRE_FIELD_COPY(generated_locally));
wire::object(dest,
WIRE_FIELD_COPY(new_address),
WIRE_FIELD_COPY(generated_locally),
WIRE_FIELD_COPY(lookahead)
);
}
void rpc::read_bytes(wire::json_reader& source, provision_subaddrs_request& self)

10
src/rpc/light_wallet.h
View File

@@ -136,6 +136,7 @@ namespace rpc
start_height(0),
transaction_height(0),
blockchain_height(0),
lookahead_fail(0),
spent_outputs(),
rates(common_error::kInvalidArgument)
{}
@@ -148,8 +149,10 @@ namespace rpc
std::uint64_t start_height;
std::uint64_t transaction_height;
std::uint64_t blockchain_height;
std::uint64_t lookahead_fail;
std::vector<transaction_spend> spent_outputs;
expect<lws::rates> rates;
db::address_index lookahead;
};
void write_bytes(wire::json_writer&, const get_address_info_response&);
@@ -171,7 +174,9 @@ namespace rpc
std::uint64_t start_height;
std::uint64_t transaction_height;
std::uint64_t blockchain_height;
std::uint64_t lookahead_fail;
std::vector<transaction> transactions;
db::address_index lookahead;
};
void write_bytes(wire::json_writer&, const get_address_txs_response&);
@@ -209,6 +214,7 @@ namespace rpc
std::uint64_t per_byte_fee;
std::uint64_t fee_mask;
safe_uint64 amount;
std::uint64_t lookahead_fail;
std::vector<std::pair<db::output, std::vector<crypto::key_image>>> outputs;
std::vector<std::uint64_t> fees;
crypto::secret_key user_key;
@@ -258,6 +264,7 @@ namespace rpc
import_request() = delete;
account_credentials creds;
std::uint64_t from_height;
db::address_index lookahead;
};
void read_bytes(wire::json_reader&, import_request&);
@@ -266,6 +273,7 @@ namespace rpc
import_response() = delete;
safe_uint64 import_fee;
std::string status;
db::address_index lookahead;
bool new_request;
bool request_fulfilled;
};
@@ -276,6 +284,7 @@ namespace rpc
{
login_request() = delete;
account_credentials creds;
db::address_index lookahead;
bool create_account;
bool generated_locally;
};
@@ -286,6 +295,7 @@ namespace rpc
login_response() = delete;
bool new_address;
bool generated_locally;
db::address_index lookahead;
};
void write_bytes(wire::json_writer&, login_response);

49
src/scanner.cpp
View File

@@ -290,20 +290,47 @@ namespace lws
struct subaddress_reader
{
expect<db::storage_reader> reader;
std::optional<db::storage> disk;
db::cursor::subaddress_indexes cur;
const std::uint32_t max_subaddresses;
subaddress_reader(std::optional<db::storage> const& disk, const bool enable_subaddresses)
: reader(common_error::kInvalidArgument), cur(nullptr)
subaddress_reader(std::optional<db::storage> const& disk_in, const std::uint32_t max_subaddresses)
: reader(common_error::kInvalidArgument), disk(), cur(nullptr), max_subaddresses(max_subaddresses)
{
if (disk && enable_subaddresses)
{
if (disk_in)
disk = disk_in->clone();
if (max_subaddresses)
update_reader();
}
void update_reader()
{
if (disk)
reader = disk->start_read();
if (!reader)
MERROR("Subadress lookup failure: " << reader.error().message());
}
if (!reader)
MERROR("Subadress lookup failure: " << reader.error().message());
}
};
void update_lookahead(const account& user, subaddress_reader& reader, const db::address_index& match, const db::block_id height)
{
if (!reader.disk)
throw std::runtime_error{"Bad DB handle in scanner"};
auto upserted = reader.disk->update_lookahead(user.db_address(), height, match, reader.max_subaddresses);
if (upserted)
{
if (0 < *upserted)
reader.update_reader(); // update reader after upsert added new addresses
else if (*upserted < 0)
upserted = {error::max_subaddresses};
}
if (!upserted)
MWARNING("Failed to update lookahead for " << user.address() << ": " << upserted.error());
}
void scan_transaction_base(
epee::span<lws::account> users,
const db::block_id height,
@@ -470,6 +497,8 @@ namespace lws
MERROR("Failure when doing subaddress search: " << match.error().message());
continue; // to next available active_derived
}
update_lookahead(user, reader, *match, height);
found_pub = true;
account_index = *match;
break; // additional_derivations loop
@@ -578,7 +607,7 @@ namespace lws
const auto time =
boost::numeric_cast<std::uint64_t>(std::chrono::system_clock::to_time_t(std::chrono::system_clock::now()));
subaddress_reader reader{std::optional<db::storage>{disk.clone()}, opts.enable_subaddresses};
subaddress_reader reader{std::optional<db::storage>{disk.clone()}, opts.max_subaddresses};
send_webhook sender{disk, client, self};
for (const auto& tx : parsed->txes)
scan_transaction_base(users, db::block_id::txpool, time, crypto::hash{}, tx, fake_outs, reader, null_spend{}, sender);
@@ -829,7 +858,7 @@ namespace lws
);
}
subaddress_reader reader{disk, opts.enable_subaddresses};
subaddress_reader reader{disk, opts.max_subaddresses};
db::block_difficulty::unsigned_int diff{};
const db::block_id initial_height = db::block_id(fetched->start_height);
for (auto block_data : boost::combine(blocks, indices))
@@ -1319,7 +1348,7 @@ namespace lws
{
if (has_shutdown())
MONERO_THROW(common_error::kInvalidArgument, "this has shutdown");
if (!lws_server_addr.empty() && (opts.enable_subaddresses || opts.untrusted_daemon))
if (!lws_server_addr.empty() && (opts.max_subaddresses || opts.untrusted_daemon))
MONERO_THROW(error::configuration, "Cannot use remote scanner with subaddresses or untrusted daemon");
if (lws_server_addr.empty())

2
src/scanner.h
View File

@@ -45,7 +45,7 @@ namespace lws
{
struct scanner_options
{
bool enable_subaddresses;
std::uint32_t max_subaddresses;
bool untrusted_daemon;
bool regtest;
};

9
src/server_main.cpp
View File

@@ -86,6 +86,7 @@ namespace
const command_line::arg_descriptor<bool> untrusted_daemon;
const command_line::arg_descriptor<bool> regtest;
const command_line::arg_descriptor<bool> version;
const command_line::arg_descriptor<bool> auto_accept_import;
static std::string get_default_zmq()
{
@@ -135,6 +136,7 @@ namespace
, untrusted_daemon{"untrusted-daemon", "Perform (expensive) chain-verification and PoW checks", false}
, regtest{"regtest", "Run in a regression testing mode", false}
, version{"version", "Display version and quit", false}
, auto_accept_import{"auto-accept-import", "Account import requests are automatically accepted", false}
{}
void prepare(boost::program_options::options_description& description) const
@@ -172,6 +174,7 @@ namespace
command_line::add_arg(description, untrusted_daemon);
command_line::add_arg(description, regtest);
command_line::add_arg(description, version);
command_line::add_arg(description, auto_accept_import);
}
};
@@ -274,7 +277,8 @@ namespace
webhook_verify,
command_line::get_arg(args, opts.external_bind),
command_line::get_arg(args, opts.disable_admin_auth),
command_line::get_arg(args, opts.auto_accept_creation)
command_line::get_arg(args, opts.auto_accept_creation),
command_line::get_arg(args, opts.auto_accept_import)
},
command_line::get_arg(args, opts.daemon_rpc),
command_line::get_arg(args, opts.daemon_sub),
@@ -332,7 +336,6 @@ namespace
auto client = scanner.sync(ctx.connect().value(), prog.untrusted_daemon).value();
const auto enable_subaddresses = bool(prog.rest_config.max_subaddresses);
lws::rest_server server{
epee::to_span(prog.rest_servers), prog.admin_rest_servers, std::move(disk), std::move(client), std::move(prog.rest_config)
};
@@ -347,7 +350,7 @@ namespace
prog.scan_threads,
std::move(prog.lws_server_addr),
std::move(prog.lws_server_pass),
lws::scanner_options{enable_subaddresses, prog.untrusted_daemon, prog.regtest}
lws::scanner_options{prog.rest_config.max_subaddresses, prog.untrusted_daemon, prog.regtest}
);
}
} // anonymous

5
src/wire/wrapper/array.h
View File

@@ -152,6 +152,11 @@ namespace wire
return {std::move(value)};
}
template<typename T, typename C>
inline bool operator==(const array_<T, C>& lhs, const array_<T, C>& rhs)
{ return lhs.get_container() == rhs.get_container(); }
/* Do not register with `is_optional_on_empty` trait, this allows selection
on whether an array is mandatory on wire. */